diff --git a/Makefile b/Makefile index f197d83b1a..e4c348b4b5 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ endif ifneq ($(CXX_BUILD), 1) ifneq ($(C89_BUILD),) - CFLAGS += -std=c89 -ansi -pedantic -Werror=pedantic -Wno-long-long -Werror=declaration-after-statement + CFLAGS += -std=c89 -ansi -pedantic -Werror=pedantic -Wno-long-long -Werror=declaration-after-statement -Wno-variadic-macros else ifeq ($(HAVE_C99), 1) CFLAGS += $(C99_CFLAGS) endif diff --git a/Makefile.common b/Makefile.common index 4596b84651..a44606b7de 100644 --- a/Makefile.common +++ b/Makefile.common @@ -1971,6 +1971,22 @@ ifeq ($(HAVE_7ZIP),1) $(7ZOBJ) endif +ifeq ($(HAVE_ZSTD),1) + INCLUDE_DIRS += -I$(DEPS_DIR)/zstd/lib + DEFINES += -DHAVE_ZSTD -DZSTD_DISABLE_ASM + ZSOBJ = $(DEPS_DIR)/zstd/lib/common/entropy_common.o \ + $(DEPS_DIR)/zstd/lib/common/error_private.o \ + $(DEPS_DIR)/zstd/lib/common/fse_decompress.o \ + $(DEPS_DIR)/zstd/lib/common/zstd_common.o \ + $(DEPS_DIR)/zstd/lib/common/xxhash.o \ + $(DEPS_DIR)/zstd/lib/decompress/huf_decompress.o \ + $(DEPS_DIR)/zstd/lib/decompress/zstd_ddict.o \ + $(DEPS_DIR)/zstd/lib/decompress/zstd_decompress.o \ + $(DEPS_DIR)/zstd/lib/decompress/zstd_decompress_block.o + + OBJ += $(ZSOBJ) +endif + ifeq ($(HAVE_IBXM), 1) DEFINES += -DHAVE_IBXM OBJ += $(DEPS_DIR)/ibxm/ibxm.o @@ -2123,6 +2139,13 @@ ifeq ($(HAVE_ZLIB_COMMON), 1) DEFINES += -Dcdlz_codec_init=retroarch_internal_cdlz_codec_init DEFINES += -Dcdlz_codec_free=retroarch_internal_cdlz_codec_free DEFINES += -Dcdlz_codec_decompress=retroarch_internal_cdlz_codec_decompress + + DEFINES += -Dzstd_codec_init=retroarch_internal_zstd_codec_init + DEFINES += -Dzstd_codec_free=retroarch_internal_zstd_codec_free + DEFINES += -Dzstd_codec_decompress=retroarch_internal_zstd_codec_decompress + DEFINES += -Dcdzs_codec_init=retroarch_internal_cdzs_codec_init + DEFINES += -Dcdzs_codec_free=retroarch_internal_cdzs_codec_free + DEFINES += -Dcdzs_codec_decompress=retroarch_internal_cdzs_codec_decompress endif OBJ += $(LIBRETRO_COMM_DIR)/formats/libchdr/libchdr_bitstream.o \ $(LIBRETRO_COMM_DIR)/formats/libchdr/libchdr_cdrom.o \ @@ -2139,6 +2162,10 @@ ifeq ($(HAVE_ZLIB_COMMON), 1) ifeq ($(HAVE_7ZIP), 1) OBJ += $(LIBRETRO_COMM_DIR)/formats/libchdr/libchdr_lzma.o endif + + ifeq ($(HAVE_ZSTD), 1) + OBJ += $(LIBRETRO_COMM_DIR)/formats/libchdr/libchdr_zstd.o + endif endif endif diff --git a/config.features.h b/config.features.h index 8961c58e66..8e022251f3 100644 --- a/config.features.h +++ b/config.features.h @@ -284,6 +284,12 @@ #define SUPPORTS_7ZIP false #endif +#ifdef HAVE_ZSTD +#define SUPPORTS_ZSTD true +#else +#define SUPPORTS_ZSTD false +#endif + #ifdef HAVE_DYLIB #define SUPPORTS_DYLIB true #else diff --git a/deps/dr/dr_flac.h b/deps/dr/dr_flac.h index fd54402327..14324cf37c 100644 --- a/deps/dr/dr_flac.h +++ b/deps/dr/dr_flac.h @@ -1,5455 +1,12536 @@ -#ifndef dr_flac_h -#define dr_flac_h - -#define DR_FLAC_NO_STDIO - -#include -#include -#include - -typedef int8_t drflac_int8; -typedef uint8_t drflac_uint8; -typedef int16_t drflac_int16; -typedef uint16_t drflac_uint16; -typedef int32_t drflac_int32; -typedef uint32_t drflac_uint32; -typedef int64_t drflac_int64; -typedef uint64_t drflac_uint64; -typedef drflac_uint8 drflac_bool8; -typedef drflac_uint32 drflac_bool32; -#define DRFLAC_TRUE 1 -#define DRFLAC_FALSE 0 - -/* As data is read from the client it is placed into an internal buffer for fast access. This controls the - * size of that buffer. Larger values means more speed, but also more memory. In my testing there is diminishing - * returns after about 4KB, but you can fiddle with this to suit your own needs. Must be a multiple of 8. - */ -#ifndef DR_FLAC_BUFFER_SIZE -#define DR_FLAC_BUFFER_SIZE 4096 -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* Check if we can enable 64-bit optimizations. */ -#if defined(_WIN64) -#define DRFLAC_64BIT -#endif - -#if defined(__GNUC__) -#if defined(__x86_64__) || defined(__ppc64__) -#define DRFLAC_64BIT -#endif -#endif - -#ifdef DRFLAC_64BIT -typedef drflac_uint64 drflac_cache_t; -#else -typedef drflac_uint32 drflac_cache_t; -#endif - -/* The various metadata block types. */ -#define DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO 0 -#define DRFLAC_METADATA_BLOCK_TYPE_PADDING 1 -#define DRFLAC_METADATA_BLOCK_TYPE_APPLICATION 2 -#define DRFLAC_METADATA_BLOCK_TYPE_SEEKTABLE 3 -#define DRFLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT 4 -#define DRFLAC_METADATA_BLOCK_TYPE_CUESHEET 5 -#define DRFLAC_METADATA_BLOCK_TYPE_PICTURE 6 -#define DRFLAC_METADATA_BLOCK_TYPE_INVALID 127 - -/* The various picture types specified in the PICTURE block. */ -#define DRFLAC_PICTURE_TYPE_OTHER 0 -#define DRFLAC_PICTURE_TYPE_FILE_ICON 1 -#define DRFLAC_PICTURE_TYPE_OTHER_FILE_ICON 2 -#define DRFLAC_PICTURE_TYPE_COVER_FRONT 3 -#define DRFLAC_PICTURE_TYPE_COVER_BACK 4 -#define DRFLAC_PICTURE_TYPE_LEAFLET_PAGE 5 -#define DRFLAC_PICTURE_TYPE_MEDIA 6 -#define DRFLAC_PICTURE_TYPE_LEAD_ARTIST 7 -#define DRFLAC_PICTURE_TYPE_ARTIST 8 -#define DRFLAC_PICTURE_TYPE_CONDUCTOR 9 -#define DRFLAC_PICTURE_TYPE_BAND 10 -#define DRFLAC_PICTURE_TYPE_COMPOSER 11 -#define DRFLAC_PICTURE_TYPE_LYRICIST 12 -#define DRFLAC_PICTURE_TYPE_RECORDING_LOCATION 13 -#define DRFLAC_PICTURE_TYPE_DURING_RECORDING 14 -#define DRFLAC_PICTURE_TYPE_DURING_PERFORMANCE 15 -#define DRFLAC_PICTURE_TYPE_SCREEN_CAPTURE 16 -#define DRFLAC_PICTURE_TYPE_BRIGHT_COLORED_FISH 17 -#define DRFLAC_PICTURE_TYPE_ILLUSTRATION 18 -#define DRFLAC_PICTURE_TYPE_BAND_LOGOTYPE 19 -#define DRFLAC_PICTURE_TYPE_PUBLISHER_LOGOTYPE 20 - -typedef enum -{ - drflac_container_native, - drflac_container_ogg, - drflac_container_unknown -} drflac_container; - -typedef enum -{ - drflac_seek_origin_start, - drflac_seek_origin_current -} drflac_seek_origin; - -/* Packing is important on this structure because we map this directly to the raw data within the SEEKTABLE metadata block. */ -#pragma pack(2) -typedef struct -{ - drflac_uint64 firstSample; - drflac_uint64 frameOffset; /* The offset from the first byte of the header of the first frame. */ - drflac_uint16 sampleCount; -} drflac_seekpoint; -#pragma pack() - -typedef struct -{ - drflac_uint16 minBlockSize; - drflac_uint16 maxBlockSize; - drflac_uint32 minFrameSize; - drflac_uint32 maxFrameSize; - drflac_uint32 sampleRate; - drflac_uint8 channels; - drflac_uint8 bitsPerSample; - drflac_uint64 totalSampleCount; - drflac_uint8 md5[16]; -} drflac_streaminfo; - -typedef struct -{ - /* The metadata type. Use this to know how to interpret the data below. */ - drflac_uint32 type; - - /* A pointer to the raw data. This points to a temporary buffer so don't hold on to it. It's best to - * not modify the contents of this buffer. Use the structures below for more meaningful and structured - * information about the metadata. It's possible for this to be null. - */ - const void* pRawData; - - /* The size in bytes of the block and the buffer pointed to by pRawData if it's non-NULL. */ - drflac_uint32 rawDataSize; - - union - { - drflac_streaminfo streaminfo; - - struct - { - int unused; - } padding; - - struct - { - drflac_uint32 id; - const void* pData; - drflac_uint32 dataSize; - } application; - - struct - { - drflac_uint32 seekpointCount; - const drflac_seekpoint* pSeekpoints; - } seektable; - - struct - { - drflac_uint32 vendorLength; - const char* vendor; - drflac_uint32 commentCount; - const char* comments; - } vorbis_comment; - - struct - { - char catalog[128]; - drflac_uint64 leadInSampleCount; - drflac_bool32 isCD; - drflac_uint8 trackCount; - const drflac_uint8* pTrackData; - } cuesheet; - - struct - { - drflac_uint32 type; - drflac_uint32 mimeLength; - const char* mime; - drflac_uint32 descriptionLength; - const char* description; - drflac_uint32 width; - drflac_uint32 height; - drflac_uint32 colorDepth; - drflac_uint32 indexColorCount; - drflac_uint32 pictureDataSize; - const drflac_uint8* pPictureData; - } picture; - } data; -} drflac_metadata; - - -/* Callback for when data needs to be read from the client. - * - * pUserData [in] The user data that was passed to drflac_open() and family. - * pBufferOut [out] The output buffer. - * bytesToRead [in] The number of bytes to read. - * - * Returns the number of bytes actually read. - * - * A return value of less than bytesToRead indicates the end of the stream. Do _not_ return from this callback until - * either the entire bytesToRead is filled or you have reached the end of the stream. - */ -typedef size_t (* drflac_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead); - -/* Callback for when data needs to be seeked. - * - * pUserData [in] The user data that was passed to drflac_open() and family. - * offset [in] The number of bytes to move, relative to the origin. Will never be negative. - * origin [in] The origin of the seek - the current position or the start of the stream. - * - * Returns whether or not the seek was successful. - * - * The offset will never be negative. Whether or not it is relative to the beginning or current position is determined - * by the "origin" parameter which will be either drflac_seek_origin_start or drflac_seek_origin_current. - */ -typedef drflac_bool32 (* drflac_seek_proc)(void* pUserData, int offset, drflac_seek_origin origin); - -/* Callback for when a metadata block is read. - * - * pUserData [in] The user data that was passed to drflac_open() and family. - * pMetadata [in] A pointer to a structure containing the data of the metadata block. - * - * Use pMetadata->type to determine which metadata block is being handled and how to read the data. - */ -typedef void (* drflac_meta_proc)(void* pUserData, drflac_metadata* pMetadata); - - -/* Structure for internal use. Only used for decoders opened with drflac_open_memory. */ -typedef struct -{ - const drflac_uint8* data; - size_t dataSize; - size_t currentReadPos; -} drflac__memory_stream; - -/* Structure for internal use. Used for bit streaming. */ -typedef struct -{ - /* The function to call when more data needs to be read. */ - drflac_read_proc onRead; - - /* The function to call when the current read position needs to be moved. */ - drflac_seek_proc onSeek; - - /* The user data to pass around to onRead and onSeek. */ - void* pUserData; - - - /* The number of unaligned bytes in the L2 cache. This will always be 0 until the end of the stream is hit. At the end of the - * stream there will be a number of bytes that don't cleanly fit in an L1 cache line, so we use this variable to know whether - * or not the bistreamer needs to run on a slower path to read those last bytes. This will never be more than sizeof(drflac_cache_t). - */ - size_t unalignedByteCount; - - /* The content of the unaligned bytes. */ - drflac_cache_t unalignedCache; - - /* The index of the next valid cache line in the "L2" cache. */ - drflac_uint32 nextL2Line; - - /* The number of bits that have been consumed by the cache. This is used to determine how many valid bits are remaining. */ - drflac_uint32 consumedBits; - - /* The cached data which was most recently read from the client. There are two levels of cache. Data flows as such: - * Client -> L2 -> L1. The L2 -> L1 movement is aligned and runs on a fast path in just a few instructions. */ - drflac_cache_t cacheL2[DR_FLAC_BUFFER_SIZE/sizeof(drflac_cache_t)]; - drflac_cache_t cache; - - /* CRC-16. This is updated whenever bits are read from the bit stream. Manually set this to 0 to reset the CRC. For FLAC, this - * is reset to 0 at the beginning of each frame. */ - drflac_uint16 crc16; - drflac_cache_t crc16Cache; /* A cache for optimizing CRC calculations. This is filled when when the L1 cache is reloaded. */ - drflac_uint32 crc16CacheIgnoredBytes; /* The number of bytes to ignore when updating the CRC-16 from the CRC-16 cache. */ -} drflac_bs; - -typedef struct -{ - /* The type of the subframe: SUBFRAME_CONSTANT, SUBFRAME_VERBATIM, SUBFRAME_FIXED or SUBFRAME_LPC. */ - drflac_uint8 subframeType; - - /* The number of wasted bits per sample as specified by the sub-frame header. */ - drflac_uint8 wastedBitsPerSample; - - /* The order to use for the prediction stage for SUBFRAME_FIXED and SUBFRAME_LPC. */ - drflac_uint8 lpcOrder; - - /* The number of bits per sample for this subframe. This is not always equal to the current frame's bit per sample because - * an extra bit is required for side channels when interchannel decorrelation is being used. */ - drflac_uint32 bitsPerSample; - - /* A pointer to the buffer containing the decoded samples in the subframe. This pointer is an offset from drflac::pExtraData. Note that - * it's a signed 32-bit integer for each value. */ - drflac_int32* pDecodedSamples; -} drflac_subframe; - -typedef struct -{ - /* If the stream uses variable block sizes, this will be set to the index of the first sample. If fixed block sizes are used, this will - * always be set to 0. */ - drflac_uint64 sampleNumber; - - /* If the stream uses fixed block sizes, this will be set to the frame number. If variable block sizes are used, this will always be 0. */ - drflac_uint32 frameNumber; - - /* The sample rate of this frame. */ - drflac_uint32 sampleRate; - - /* The number of samples in each sub-frame within this frame. */ - drflac_uint16 blockSize; - - /* The channel assignment of this frame. This is not always set to the channel count. If interchannel decorrelation is being used this - * will be set to DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE, DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE or DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE. */ - drflac_uint8 channelAssignment; - - /* The number of bits per sample within this frame. */ - drflac_uint8 bitsPerSample; - - /* The frame's CRC. */ - drflac_uint8 crc8; -} drflac_frame_header; - -typedef struct -{ - /* The header. */ - drflac_frame_header header; - - /* The number of samples left to be read in this frame. This is initially set to the block size multiplied by the channel count. As samples - * are read, this will be decremented. When it reaches 0, the decoder will see this frame as fully consumed and load the next frame. - */ - drflac_uint32 samplesRemaining; - - /* The list of sub-frames within the frame. There is one sub-frame for each channel, and there's a maximum of 8 channels. */ - drflac_subframe subframes[8]; -} drflac_frame; - -typedef struct -{ - /* The function to call when a metadata block is read. */ - drflac_meta_proc onMeta; - - /* The user data posted to the metadata callback function. */ - void* pUserDataMD; - - - /* The sample rate. Will be set to something like 44100. */ - drflac_uint32 sampleRate; - - /* The number of channels. This will be set to 1 for monaural streams, 2 for stereo, etc. Maximum 8. This is set based on the - * value specified in the STREAMINFO block. */ - drflac_uint8 channels; - - /* The bits per sample. Will be set to somthing like 16, 24, etc. */ - drflac_uint8 bitsPerSample; - - /* The maximum block size, in samples. This number represents the number of samples in each channel (not combined). */ - drflac_uint16 maxBlockSize; - - /* The total number of samples making up the stream. This includes every channel. For example, if the stream has 2 channels, - * with each channel having a total of 4096, this value will be set to 2*4096 = 8192. Can be 0 in which case it's still a - * valid stream, but just means the total sample count is unknown. Likely the case with streams like internet radio. */ - drflac_uint64 totalSampleCount; - - - /* The container type. This is set based on whether or not the decoder was opened from a native or Ogg stream. */ - drflac_container container; - - /* The position of the seektable in the file. */ - drflac_uint64 seektablePos; - - /* The size of the seektable. */ - drflac_uint32 seektableSize; - - - /* Information about the frame the decoder is currently sitting on. */ - drflac_frame currentFrame; - - /* The position of the first frame in the stream. This is only ever used for seeking. */ - drflac_uint64 firstFramePos; - - - /* A hack to avoid a malloc() when opening a decoder with drflac_open_memory(). */ - drflac__memory_stream memoryStream; - - - /* A pointer to the decoded sample data. This is an offset of pExtraData. */ - drflac_int32* pDecodedSamples; - - /* Internal use only. Only used with Ogg containers. Points to a drflac_oggbs object. This is an offset of pExtraData. */ - void* _oggbs; - - /* The bit streamer. The raw FLAC data is fed through this object. */ - drflac_bs bs; - - /* Variable length extra data. We attach this to the end of the object so we can avoid unnecessary mallocs. */ - drflac_uint8 pExtraData[1]; -} drflac; - - -/* Opens a FLAC decoder. - * - * onRead [in] The function to call when data needs to be read from the client. - * onSeek [in] The function to call when the read position of the client data needs to move. - * pUserData [in, optional] A pointer to application defined data that will be passed to onRead and onSeek. - * - * Returns a pointer to an object representing the decoder. - * - * Close the decoder with drflac_close(). - * - * This function will automatically detect whether or not you are attempting to open a native or Ogg encapsulated - * FLAC, both of which should work seamlessly without any manual intervention. Ogg encapsulation also works with - * multiplexed streams which basically means it can play FLAC encoded audio tracks in videos. - * - * This is the lowest level function for opening a FLAC stream. You can also use drflac_open_file() and drflac_open_memory() - * to open the stream from a file or from a block of memory respectively. - * - * The STREAMINFO block must be present for this to succeed. Use drflac_open_relaxed() to open a FLAC stream where - * the header may not be present. - * - * See also: drflac_open_file(), drflac_open_memory(), drflac_open_with_metadata(), drflac_close() - */ -drflac* drflac_open(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData); - -/* The same as drflac_open(), except attempts to open the stream even when a header block is not present. - * - * Because the header is not necessarily available, the caller must explicitly define the container (Native or Ogg). Do - * not set this to drflac_container_unknown - that is for internal use only. - * - * Opening in relaxed mode will continue reading data from onRead until it finds a valid frame. If a frame is never - * found it will continue forever. To abort, force your onRead callback to return 0, which dr_flac will use as an - * indicator that the end of the stream was found. */ -drflac* drflac_open_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_container container, void* pUserData); - -/* Opens a FLAC decoder and notifies the caller of the metadata chunks (album art, etc.). - * - * onRead [in] The function to call when data needs to be read from the client. - * onSeek [in] The function to call when the read position of the client data needs to move. - * onMeta [in] The function to call for every metadata block. - * pUserData [in, optional] A pointer to application defined data that will be passed to onRead, onSeek and onMeta. - * - * Returns a pointer to an object representing the decoder. - * - * Close the decoder with drflac_close(). - * - * This is slower than drflac_open(), so avoid this one if you don't need metadata. Internally, this will do a DRFLAC_MALLOC() - * and DRFLAC_FREE() for every metadata block except for STREAMINFO and PADDING blocks. - * - * The caller is notified of the metadata via the onMeta callback. All metadata blocks will be handled before the function - * returns. - * - * The STREAMINFO block must be present for this to succeed. Use drflac_open_with_metadata_relaxed() to open a FLAC - * stream where the header may not be present. - * - * Note that this will behave inconsistently with drflac_open() if the stream is an Ogg encapsulated stream and a metadata - * block is corrupted. This is due to the way the Ogg stream recovers from corrupted pages. When drflac_open_with_metadata() - * is being used, the open routine will try to read the contents of the metadata block, whereas drflac_open() will simply - * seek past it (for the sake of efficiency). This inconsistency can result in different samples being returned depending on - * whether or not the stream is being opened with metadata. - * - * See also: drflac_open_file_with_metadata(), drflac_open_memory_with_metadata(), drflac_open(), drflac_close() - */ -drflac* drflac_open_with_metadata(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData); - -/* The same as drflac_open_with_metadata(), except attemps to open the stream even when a header block is not present. - * - * See also: drflac_open_with_metadata(), drflac_open_relaxed() - */ -drflac* drflac_open_with_metadata_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData); - -/* Closes the given FLAC decoder. - * - * pFlac [in] The decoder to close. - * - * This will destroy the decoder object. */ -void drflac_close(drflac* pFlac); - - -/* Reads sample data from the given FLAC decoder, output as interleaved signed 32-bit PCM. - * - * pFlac [in] The decoder. - * samplesToRead [in] The number of samples to read. - * pBufferOut [out, optional] A pointer to the buffer that will receive the decoded samples. - * - * Returns the number of samples actually read. - * - * pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of samples - * seeked. */ -drflac_uint64 drflac_read_s32(drflac* pFlac, drflac_uint64 samplesToRead, drflac_int32* pBufferOut); - -/* Same as drflac_read_s32(), except outputs samples as 16-bit integer PCM rather than 32-bit. - * - * pFlac [in] The decoder. - * samplesToRead [in] The number of samples to read. - * pBufferOut [out, optional] A pointer to the buffer that will receive the decoded samples. - * - * Returns the number of samples actually read. - * - * pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of samples - * seeked. - * - * Note that this is lossy for streams where the bits per sample is larger than 16. - */ -drflac_uint64 drflac_read_s16(drflac* pFlac, drflac_uint64 samplesToRead, drflac_int16* pBufferOut); - -/* Same as drflac_read_s32(), except outputs samples as 32-bit floating-point PCM. - * - * pFlac [in] The decoder. - * samplesToRead [in] The number of samples to read. - * pBufferOut [out, optional] A pointer to the buffer that will receive the decoded samples. - * - * Returns the number of samples actually read. - * - * pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of samples - * seeked. - * - * Note that this should be considered lossy due to the nature of floating point numbers not being able to exactly - * represent every possible number. - */ -drflac_uint64 drflac_read_f32(drflac* pFlac, drflac_uint64 samplesToRead, float* pBufferOut); - -/* Seeks to the sample at the given index. - * - * pFlac [in] The decoder. - * sampleIndex [in] The index of the sample to seek to. See notes below. - * - * Returns DRFLAC_TRUE if successful; DRFLAC_FALSE otherwise. - * - * The sample index is based on interleaving. In a stereo stream, for example, the sample at index 0 is the first sample - * in the left channel; the sample at index 1 is the first sample on the right channel, and so on. - * - * When seeking, you will likely want to ensure it's rounded to a multiple of the channel count. You can do this with - * something like drflac_seek_to_sample(pFlac, (mySampleIndex + (mySampleIndex % pFlac->channels))) - */ -drflac_bool32 drflac_seek_to_sample(drflac* pFlac, drflac_uint64 sampleIndex); - - - -#ifndef DR_FLAC_NO_STDIO -/* Opens a FLAC decoder from the file at the given path. - * - * filename [in] The path of the file to open, either absolute or relative to the current directory. - * - * Returns a pointer to an object representing the decoder. - * - * Close the decoder with drflac_close(). - * - * This will hold a handle to the file until the decoder is closed with drflac_close(). Some platforms will restrict the - * number of files a process can have open at any given time, so keep this mind if you have many decoders open at the - * same time. - * - * See also: drflac_open(), drflac_open_file_with_metadata(), drflac_close() - */ -drflac* drflac_open_file(const char* filename); - -/* Opens a FLAC decoder from the file at the given path and notifies the caller of the metadata chunks (album art, etc.) - * - * Look at the documentation for drflac_open_with_metadata() for more information on how metadata is handled. - */ -drflac* drflac_open_file_with_metadata(const char* filename, drflac_meta_proc onMeta, void* pUserData); -#endif - -/* Opens a FLAC decoder from a pre-allocated block of memory - * - * This does not create a copy of the data. It is up to the application to ensure the buffer remains valid for - * the lifetime of the decoder. - */ -drflac* drflac_open_memory(const void* data, size_t dataSize); - -/* Opens a FLAC decoder from a pre-allocated block of memory and notifies the caller of the metadata chunks (album art, etc.) - * - * Look at the documentation for drflac_open_with_metadata() for more information on how metadata is handled. - */ -drflac* drflac_open_memory_with_metadata(const void* data, size_t dataSize, drflac_meta_proc onMeta, void* pUserData); - -/* High Level APIs */ - -/* Opens a FLAC stream from the given callbacks and fully decodes it in a single operation. The return value is a - * pointer to the sample data as interleaved signed 32-bit PCM. The returned data must be freed with DRFLAC_FREE(). - * - * Sometimes a FLAC file won't keep track of the total sample count. In this situation the function will continuously - * read samples into a dynamically sized buffer on the heap until no samples are left. - * - * Do not call this function on a broadcast type of stream (like internet radio streams and whatnot). - */ -drflac_int32* drflac_open_and_decode_s32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_s32(), except returns signed 16-bit integer samples. */ -drflac_int16* drflac_open_and_decode_s16(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_s32(), except returns 32-bit floating-point samples. */ -float* drflac_open_and_decode_f32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -#ifndef DR_FLAC_NO_STDIO -/* Same as drflac_open_and_decode_s32() except opens the decoder from a file. */ -drflac_int32* drflac_open_and_decode_file_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_file_s32(), except returns signed 16-bit integer samples. */ -drflac_int16* drflac_open_and_decode_file_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_file_f32(), except returns 32-bit floating-point samples. */ -float* drflac_open_and_decode_file_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); -#endif - -/* Same as drflac_open_and_decode_s32() except opens the decoder from a block of memory. */ -drflac_int32* drflac_open_and_decode_memory_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_memory_s32(), except returns signed 16-bit integer samples. */ -drflac_int16* drflac_open_and_decode_memory_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Same as drflac_open_and_decode_memory_s32(), except returns 32-bit floating-point samples. */ -float* drflac_open_and_decode_memory_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount); - -/* Frees memory that was allocated internally by dr_flac. */ -void drflac_free(void* p); - - -/* Structure representing an iterator for vorbis comments in a VORBIS_COMMENT metadata block. */ -typedef struct -{ - drflac_uint32 countRemaining; - const char* pRunningData; -} drflac_vorbis_comment_iterator; - -/* Initializes a vorbis comment iterator. This can be used for iterating over the vorbis comments in a VORBIS_COMMENT - * metadata block. */ -void drflac_init_vorbis_comment_iterator(drflac_vorbis_comment_iterator* pIter, drflac_uint32 commentCount, const char* pComments); - -/* Goes to the next vorbis comment in the given iterator. If null is returned it means there are no more comments. The - * returned string is NOT null terminated. */ -const char* drflac_next_vorbis_comment(drflac_vorbis_comment_iterator* pIter, drflac_uint32* pCommentLengthOut); - - - -#ifdef __cplusplus -} -#endif -#endif /* dr_flac_h */ - - -/* - * - * IMPLEMENTATION - * - */ -#ifdef DR_FLAC_IMPLEMENTATION -#include -#include - -/* CPU architecture. */ -#if defined(__x86_64__) || defined(_M_X64) -#define DRFLAC_X64 -#elif defined(__i386) || defined(_M_IX86) -#define DRFLAC_X86 -#elif defined(__arm__) || defined(_M_ARM) -#define DRFLAC_ARM -#endif - -/* Compile-time CPU feature support. */ -#if !defined(DR_FLAC_NO_SIMD) && (defined(DRFLAC_X86) || defined(DRFLAC_X64)) - #if defined(_MSC_VER) && !defined(__clang__) - #if _MSC_VER >= 1400 - #include - static void drflac__cpuid(int info[4], int fid) - { - __cpuid(info, fid); - } - #else - #define DRFLAC_NO_CPUID - #endif - #else - #if (defined(__GNUC__) || defined(__clang__)) && !defined(__ANDROID__) - static void drflac__cpuid(int info[4], int fid) - { - __asm__ __volatile__ ( - "cpuid" : "=a"(info[0]), "=b"(info[1]), "=c"(info[2]), "=d"(info[3]) : "a"(fid), "c"(0) - ); - } - #else - #define DRFLAC_NO_CPUID - #endif - #endif -#else -#define DRFLAC_NO_CPUID -#endif - -#ifdef __linux__ - #ifndef _BSD_SOURCE - #define _BSD_SOURCE - #endif - #ifndef _DEFAULT_SOURCE - #define _DEFAULT_SOURCE - #endif - #ifndef __USE_BSD - #define __USE_BSD - #endif - #include -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1500 && (defined(DRFLAC_X86) || defined(DRFLAC_X64)) -#define DRFLAC_HAS_LZCNT_INTRINSIC -#elif (defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))) -#define DRFLAC_HAS_LZCNT_INTRINSIC -#elif defined(__clang__) - #if __has_builtin(__builtin_clzll) || __has_builtin(__builtin_clzl) - #define DRFLAC_HAS_LZCNT_INTRINSIC - #endif -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1300 -#define DRFLAC_HAS_BYTESWAP_INTRINSIC -#elif defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -#define DRFLAC_HAS_BYTESWAP_INTRINSIC -#elif defined(__clang__) - #if __has_builtin(__builtin_bswap16) && __has_builtin(__builtin_bswap32) && __has_builtin(__builtin_bswap64) - #define DRFLAC_HAS_BYTESWAP_INTRINSIC - #endif -#endif - -/* Standard library stuff. */ -#ifndef DRFLAC_ASSERT -#include -#define DRFLAC_ASSERT(expression) assert(expression) -#endif -#ifndef DRFLAC_MALLOC -#define DRFLAC_MALLOC(sz) malloc((sz)) -#endif -#ifndef DRFLAC_REALLOC -#define DRFLAC_REALLOC(p, sz) realloc((p), (sz)) -#endif -#ifndef DRFLAC_FREE -#define DRFLAC_FREE(p) free((p)) -#endif -#ifndef DRFLAC_COPY_MEMORY -#define DRFLAC_COPY_MEMORY(dst, src, sz) memcpy((dst), (src), (sz)) -#endif -#ifndef DRFLAC_ZERO_MEMORY -#define DRFLAC_ZERO_MEMORY(p, sz) memset((p), 0, (sz)) -#endif - -#define DRFLAC_MAX_SIMD_VECTOR_SIZE 64 /* 64 for AVX-512 in the future. */ - -#ifdef _MSC_VER -#define DRFLAC_INLINE __forceinline -#else -#ifdef __GNUC__ -#define DRFLAC_INLINE INLINE __attribute__((always_inline)) -#else -#define DRFLAC_INLINE INLINE -#endif -#endif - -typedef drflac_int32 drflac_result; -#define DRFLAC_SUCCESS 0 -#define DRFLAC_ERROR -1 /* A generic error. */ -#define DRFLAC_INVALID_ARGS -2 -#define DRFLAC_END_OF_STREAM -128 -#define DRFLAC_CRC_MISMATCH -129 - -#define DRFLAC_SUBFRAME_CONSTANT 0 -#define DRFLAC_SUBFRAME_VERBATIM 1 -#define DRFLAC_SUBFRAME_FIXED 8 -#define DRFLAC_SUBFRAME_LPC 32 -#define DRFLAC_SUBFRAME_RESERVED 255 - -#define DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE 0 -#define DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2 1 - -#define DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT 0 -#define DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE 8 -#define DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE 9 -#define DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE 10 - - -#define drflac_align(x, a) ((((x) + (a) - 1) / (a)) * (a)) -#define drflac_assert DRFLAC_ASSERT -#define drflac_copy_memory DRFLAC_COPY_MEMORY -#define drflac_zero_memory DRFLAC_ZERO_MEMORY - -/* CPU caps. */ -static drflac_bool32 drflac__gIsLZCNTSupported = DRFLAC_FALSE; -#ifndef DRFLAC_NO_CPUID -static drflac_bool32 drflac__gIsSSE42Supported = DRFLAC_FALSE; -static void drflac__init_cpu_caps(void) -{ - int info[4] = {0}; - - /* LZCNT */ - drflac__cpuid(info, 0x80000001); - drflac__gIsLZCNTSupported = (info[2] & (1 << 5)) != 0; - - /* SSE4.2 */ - drflac__cpuid(info, 1); - drflac__gIsSSE42Supported = (info[2] & (1 << 19)) != 0; -} -#endif - - -/* Endian Management */ -static DRFLAC_INLINE drflac_bool32 drflac__is_little_endian(void) -{ -#if defined(DRFLAC_X86) || defined(DRFLAC_X64) - return DRFLAC_TRUE; -#else - int n = 1; - return (*(char*)&n) == 1; -#endif -} - -static DRFLAC_INLINE drflac_uint16 drflac__swap_endian_uint16(drflac_uint16 n) -{ -#ifdef DRFLAC_HAS_BYTESWAP_INTRINSIC - #if defined(_MSC_VER) - return _byteswap_ushort(n); - #elif defined(__GNUC__) || defined(__clang__) - return __builtin_bswap16(n); - #else - #error "This compiler does not support the byte swap intrinsic." - #endif -#else - return ((n & 0xFF00) >> 8) | - ((n & 0x00FF) << 8); -#endif -} - -static DRFLAC_INLINE drflac_uint32 drflac__swap_endian_uint32(drflac_uint32 n) -{ -#ifdef DRFLAC_HAS_BYTESWAP_INTRINSIC - #if defined(_MSC_VER) - return _byteswap_ulong(n); - #elif defined(__GNUC__) || defined(__clang__) - return __builtin_bswap32(n); - #else - #error "This compiler does not support the byte swap intrinsic." - #endif -#else - return ((n & 0xFF000000) >> 24) | - ((n & 0x00FF0000) >> 8) | - ((n & 0x0000FF00) << 8) | - ((n & 0x000000FF) << 24); -#endif -} - -static DRFLAC_INLINE drflac_uint64 drflac__swap_endian_uint64(drflac_uint64 n) -{ -#ifdef DRFLAC_HAS_BYTESWAP_INTRINSIC - #if defined(_MSC_VER) - return _byteswap_uint64(n); - #elif defined(__GNUC__) || defined(__clang__) - return __builtin_bswap64(n); - #else - #error "This compiler does not support the byte swap intrinsic." - #endif -#else - return ((n & (drflac_uint64)0xFF00000000000000) >> 56) | - ((n & (drflac_uint64)0x00FF000000000000) >> 40) | - ((n & (drflac_uint64)0x0000FF0000000000) >> 24) | - ((n & (drflac_uint64)0x000000FF00000000) >> 8) | - ((n & (drflac_uint64)0x00000000FF000000) << 8) | - ((n & (drflac_uint64)0x0000000000FF0000) << 24) | - ((n & (drflac_uint64)0x000000000000FF00) << 40) | - ((n & (drflac_uint64)0x00000000000000FF) << 56); -#endif -} - - -static DRFLAC_INLINE drflac_uint16 drflac__be2host_16(drflac_uint16 n) -{ -#ifdef __linux__ - return be16toh(n); -#else - if (drflac__is_little_endian()) - return drflac__swap_endian_uint16(n); - - return n; -#endif -} - -static DRFLAC_INLINE drflac_uint32 drflac__be2host_32(drflac_uint32 n) -{ -#ifdef __linux__ - return be32toh(n); -#else - if (drflac__is_little_endian()) - return drflac__swap_endian_uint32(n); - - return n; -#endif -} - -static DRFLAC_INLINE drflac_uint64 drflac__be2host_64(drflac_uint64 n) -{ -#ifdef __linux__ - return be64toh(n); -#else - if (drflac__is_little_endian()) - return drflac__swap_endian_uint64(n); - - return n; -#endif -} - - -static DRFLAC_INLINE drflac_uint32 drflac__le2host_32(drflac_uint32 n) -{ -#ifdef __linux__ - return le32toh(n); -#else - if (!drflac__is_little_endian()) - return drflac__swap_endian_uint32(n); - - return n; -#endif -} - - -static DRFLAC_INLINE drflac_uint32 drflac__unsynchsafe_32(drflac_uint32 n) -{ - drflac_uint32 result = 0; - result |= (n & 0x7F000000) >> 3; - result |= (n & 0x007F0000) >> 2; - result |= (n & 0x00007F00) >> 1; - result |= (n & 0x0000007F) >> 0; - - return result; -} - -/* The CRC code below is based on this document: http://zlib.net/crc_v3.txt */ -static drflac_uint8 drflac__crc8_table[] = { - 0x00, 0x07, 0x0E, 0x09, 0x1C, 0x1B, 0x12, 0x15, 0x38, 0x3F, 0x36, 0x31, 0x24, 0x23, 0x2A, 0x2D, - 0x70, 0x77, 0x7E, 0x79, 0x6C, 0x6B, 0x62, 0x65, 0x48, 0x4F, 0x46, 0x41, 0x54, 0x53, 0x5A, 0x5D, - 0xE0, 0xE7, 0xEE, 0xE9, 0xFC, 0xFB, 0xF2, 0xF5, 0xD8, 0xDF, 0xD6, 0xD1, 0xC4, 0xC3, 0xCA, 0xCD, - 0x90, 0x97, 0x9E, 0x99, 0x8C, 0x8B, 0x82, 0x85, 0xA8, 0xAF, 0xA6, 0xA1, 0xB4, 0xB3, 0xBA, 0xBD, - 0xC7, 0xC0, 0xC9, 0xCE, 0xDB, 0xDC, 0xD5, 0xD2, 0xFF, 0xF8, 0xF1, 0xF6, 0xE3, 0xE4, 0xED, 0xEA, - 0xB7, 0xB0, 0xB9, 0xBE, 0xAB, 0xAC, 0xA5, 0xA2, 0x8F, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9D, 0x9A, - 0x27, 0x20, 0x29, 0x2E, 0x3B, 0x3C, 0x35, 0x32, 0x1F, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0D, 0x0A, - 0x57, 0x50, 0x59, 0x5E, 0x4B, 0x4C, 0x45, 0x42, 0x6F, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7D, 0x7A, - 0x89, 0x8E, 0x87, 0x80, 0x95, 0x92, 0x9B, 0x9C, 0xB1, 0xB6, 0xBF, 0xB8, 0xAD, 0xAA, 0xA3, 0xA4, - 0xF9, 0xFE, 0xF7, 0xF0, 0xE5, 0xE2, 0xEB, 0xEC, 0xC1, 0xC6, 0xCF, 0xC8, 0xDD, 0xDA, 0xD3, 0xD4, - 0x69, 0x6E, 0x67, 0x60, 0x75, 0x72, 0x7B, 0x7C, 0x51, 0x56, 0x5F, 0x58, 0x4D, 0x4A, 0x43, 0x44, - 0x19, 0x1E, 0x17, 0x10, 0x05, 0x02, 0x0B, 0x0C, 0x21, 0x26, 0x2F, 0x28, 0x3D, 0x3A, 0x33, 0x34, - 0x4E, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5C, 0x5B, 0x76, 0x71, 0x78, 0x7F, 0x6A, 0x6D, 0x64, 0x63, - 0x3E, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2C, 0x2B, 0x06, 0x01, 0x08, 0x0F, 0x1A, 0x1D, 0x14, 0x13, - 0xAE, 0xA9, 0xA0, 0xA7, 0xB2, 0xB5, 0xBC, 0xBB, 0x96, 0x91, 0x98, 0x9F, 0x8A, 0x8D, 0x84, 0x83, - 0xDE, 0xD9, 0xD0, 0xD7, 0xC2, 0xC5, 0xCC, 0xCB, 0xE6, 0xE1, 0xE8, 0xEF, 0xFA, 0xFD, 0xF4, 0xF3 -}; - -static drflac_uint16 drflac__crc16_table[] = { - 0x0000, 0x8005, 0x800F, 0x000A, 0x801B, 0x001E, 0x0014, 0x8011, - 0x8033, 0x0036, 0x003C, 0x8039, 0x0028, 0x802D, 0x8027, 0x0022, - 0x8063, 0x0066, 0x006C, 0x8069, 0x0078, 0x807D, 0x8077, 0x0072, - 0x0050, 0x8055, 0x805F, 0x005A, 0x804B, 0x004E, 0x0044, 0x8041, - 0x80C3, 0x00C6, 0x00CC, 0x80C9, 0x00D8, 0x80DD, 0x80D7, 0x00D2, - 0x00F0, 0x80F5, 0x80FF, 0x00FA, 0x80EB, 0x00EE, 0x00E4, 0x80E1, - 0x00A0, 0x80A5, 0x80AF, 0x00AA, 0x80BB, 0x00BE, 0x00B4, 0x80B1, - 0x8093, 0x0096, 0x009C, 0x8099, 0x0088, 0x808D, 0x8087, 0x0082, - 0x8183, 0x0186, 0x018C, 0x8189, 0x0198, 0x819D, 0x8197, 0x0192, - 0x01B0, 0x81B5, 0x81BF, 0x01BA, 0x81AB, 0x01AE, 0x01A4, 0x81A1, - 0x01E0, 0x81E5, 0x81EF, 0x01EA, 0x81FB, 0x01FE, 0x01F4, 0x81F1, - 0x81D3, 0x01D6, 0x01DC, 0x81D9, 0x01C8, 0x81CD, 0x81C7, 0x01C2, - 0x0140, 0x8145, 0x814F, 0x014A, 0x815B, 0x015E, 0x0154, 0x8151, - 0x8173, 0x0176, 0x017C, 0x8179, 0x0168, 0x816D, 0x8167, 0x0162, - 0x8123, 0x0126, 0x012C, 0x8129, 0x0138, 0x813D, 0x8137, 0x0132, - 0x0110, 0x8115, 0x811F, 0x011A, 0x810B, 0x010E, 0x0104, 0x8101, - 0x8303, 0x0306, 0x030C, 0x8309, 0x0318, 0x831D, 0x8317, 0x0312, - 0x0330, 0x8335, 0x833F, 0x033A, 0x832B, 0x032E, 0x0324, 0x8321, - 0x0360, 0x8365, 0x836F, 0x036A, 0x837B, 0x037E, 0x0374, 0x8371, - 0x8353, 0x0356, 0x035C, 0x8359, 0x0348, 0x834D, 0x8347, 0x0342, - 0x03C0, 0x83C5, 0x83CF, 0x03CA, 0x83DB, 0x03DE, 0x03D4, 0x83D1, - 0x83F3, 0x03F6, 0x03FC, 0x83F9, 0x03E8, 0x83ED, 0x83E7, 0x03E2, - 0x83A3, 0x03A6, 0x03AC, 0x83A9, 0x03B8, 0x83BD, 0x83B7, 0x03B2, - 0x0390, 0x8395, 0x839F, 0x039A, 0x838B, 0x038E, 0x0384, 0x8381, - 0x0280, 0x8285, 0x828F, 0x028A, 0x829B, 0x029E, 0x0294, 0x8291, - 0x82B3, 0x02B6, 0x02BC, 0x82B9, 0x02A8, 0x82AD, 0x82A7, 0x02A2, - 0x82E3, 0x02E6, 0x02EC, 0x82E9, 0x02F8, 0x82FD, 0x82F7, 0x02F2, - 0x02D0, 0x82D5, 0x82DF, 0x02DA, 0x82CB, 0x02CE, 0x02C4, 0x82C1, - 0x8243, 0x0246, 0x024C, 0x8249, 0x0258, 0x825D, 0x8257, 0x0252, - 0x0270, 0x8275, 0x827F, 0x027A, 0x826B, 0x026E, 0x0264, 0x8261, - 0x0220, 0x8225, 0x822F, 0x022A, 0x823B, 0x023E, 0x0234, 0x8231, - 0x8213, 0x0216, 0x021C, 0x8219, 0x0208, 0x820D, 0x8207, 0x0202 -}; - -static DRFLAC_INLINE drflac_uint8 drflac_crc8_byte(drflac_uint8 crc, drflac_uint8 data) -{ - return drflac__crc8_table[crc ^ data]; -} - -static DRFLAC_INLINE drflac_uint8 drflac_crc8(drflac_uint8 crc, drflac_uint32 data, drflac_uint32 count) -{ - -#ifdef DR_FLAC_NO_CRC - (void)crc; - (void)data; - (void)count; - drflac_assert(count <= 32); - return 0; -#else -#if 0 - /* REFERENCE (use of this implementation requires an explicit flush by doing "drflac_crc8(crc, 0, 8);") */ - drflac_uint8 p = 0x07; - for (int i = count-1; i >= 0; --i) - { - drflac_uint8 bit = (data & (1 << i)) >> i; - if (crc & 0x80) - crc = ((crc << 1) | bit) ^ p; - else - crc = ((crc << 1) | bit); - } - return crc; -#else - static drflac_uint64 leftoverDataMaskTable[8] = { - 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F - }; - drflac_uint32 wholeBytes = count >> 3; - drflac_uint32 leftoverBits = count - (wholeBytes*8); - drflac_uint64 leftoverDataMask = leftoverDataMaskTable[leftoverBits]; - drflac_assert(count <= 32); - - switch (wholeBytes) - { - case 4: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits))); - case 3: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits))); - case 2: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits))); - case 1: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits))); - case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ drflac__crc8_table[(crc >> (8 - leftoverBits)) ^ (data & leftoverDataMask)]; - } - return crc; -#endif -#endif -} - -static DRFLAC_INLINE drflac_uint16 drflac_crc16_byte(drflac_uint16 crc, drflac_uint8 data) -{ - return (crc << 8) ^ drflac__crc16_table[(drflac_uint8)(crc >> 8) ^ data]; -} - -static DRFLAC_INLINE drflac_uint16 drflac_crc16_bytes(drflac_uint16 crc, drflac_cache_t data, drflac_uint32 byteCount) -{ - switch (byteCount) - { -#ifdef DRFLAC_64BIT - case 8: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 56) & 0xFF)); - case 7: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 48) & 0xFF)); - case 6: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 40) & 0xFF)); - case 5: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 32) & 0xFF)); -#endif - case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 24) & 0xFF)); - case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 16) & 0xFF)); - case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 8) & 0xFF)); - case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 0) & 0xFF)); - } - - return crc; -} - -static DRFLAC_INLINE drflac_uint16 drflac_crc16__32bit(drflac_uint16 crc, drflac_uint32 data, drflac_uint32 count) -{ - -#ifdef DR_FLAC_NO_CRC - (void)crc; - (void)data; - (void)count; - drflac_assert(count <= 64); - return 0; -#else -#if 0 - /* REFERENCE (use of this implementation requires an explicit flush by doing "drflac_crc16(crc, 0, 16);") */ - drflac_uint16 p = 0x8005; - for (int i = count-1; i >= 0; --i) - { - drflac_uint16 bit = (data & (1ULL << i)) >> i; - if (r & 0x8000) - r = ((r << 1) | bit) ^ p; - else - r = ((r << 1) | bit); - } - - return crc; -#else - drflac_uint32 wholeBytes = count >> 3; - drflac_uint32 leftoverBits = count - (wholeBytes*8); - - static drflac_uint64 leftoverDataMaskTable[8] = { - 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F - }; - drflac_uint64 leftoverDataMask = leftoverDataMaskTable[leftoverBits]; - - drflac_assert(count <= 64); - - switch (wholeBytes) - { - default: - case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits))); - case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits))); - case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits))); - case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits))); - case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ drflac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)]; - } - return crc; -#endif -#endif -} - -static DRFLAC_INLINE drflac_uint16 drflac_crc16__64bit(drflac_uint16 crc, drflac_uint64 data, drflac_uint32 count) -{ - -#ifdef DR_FLAC_NO_CRC - (void)crc; - (void)data; - (void)count; - drflac_assert(count <= 64); - return 0; -#else - drflac_uint32 wholeBytes = count >> 3; - drflac_uint32 leftoverBits = count - (wholeBytes*8); - - static drflac_uint64 leftoverDataMaskTable[8] = { - 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F - }; - drflac_uint64 leftoverDataMask = leftoverDataMaskTable[leftoverBits]; - - drflac_assert(count <= 64); - - switch (wholeBytes) - { - default: - case 8: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0xFF00000000000000 << leftoverBits)) >> (56 + leftoverBits))); - case 7: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x00FF000000000000 << leftoverBits)) >> (48 + leftoverBits))); - case 6: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x0000FF0000000000 << leftoverBits)) >> (40 + leftoverBits))); - case 5: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x000000FF00000000 << leftoverBits)) >> (32 + leftoverBits))); - case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x00000000FF000000 << leftoverBits)) >> (24 + leftoverBits))); - case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x0000000000FF0000 << leftoverBits)) >> (16 + leftoverBits))); - case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x000000000000FF00 << leftoverBits)) >> ( 8 + leftoverBits))); - case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & ((drflac_uint64)0x00000000000000FF << leftoverBits)) >> ( 0 + leftoverBits))); - case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ drflac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)]; - } - return crc; -#endif -} - - -static DRFLAC_INLINE drflac_uint16 drflac_crc16(drflac_uint16 crc, drflac_cache_t data, drflac_uint32 count) -{ -#ifdef DRFLAC_64BIT - return drflac_crc16__64bit(crc, data, count); -#else - return drflac_crc16__32bit(crc, data, count); -#endif -} - - -#ifdef DRFLAC_64BIT -#define drflac__be2host__cache_line drflac__be2host_64 -#else -#define drflac__be2host__cache_line drflac__be2host_32 -#endif - -/* BIT READING ATTEMPT #2 - * - * This uses a 32- or 64-bit bit-shifted cache - as bits are read, the cache is shifted such that the first valid bit is sitting - * on the most significant bit. It uses the notion of an L1 and L2 cache (borrowed from CPU architecture), where the L1 cache - * is a 32- or 64-bit unsigned integer (depending on whether or not a 32- or 64-bit build is being compiled) and the L2 is an - * array of "cache lines", with each cache line being the same size as the L1. The L2 is a buffer of about 4KB and is where data - * from onRead() is read into. - */ -#define DRFLAC_CACHE_L1_SIZE_BYTES(bs) (sizeof((bs)->cache)) -#define DRFLAC_CACHE_L1_SIZE_BITS(bs) (sizeof((bs)->cache)*8) -#define DRFLAC_CACHE_L1_BITS_REMAINING(bs) (DRFLAC_CACHE_L1_SIZE_BITS(bs) - ((bs)->consumedBits)) -#ifdef DRFLAC_64BIT -#define DRFLAC_CACHE_L1_SELECTION_MASK(_bitCount) (~(((drflac_uint64)-1LL) >> (_bitCount))) -#else -#define DRFLAC_CACHE_L1_SELECTION_MASK(_bitCount) (~(((drflac_uint32)-1) >> (_bitCount))) -#endif -#define DRFLAC_CACHE_L1_SELECTION_SHIFT(bs, _bitCount) (DRFLAC_CACHE_L1_SIZE_BITS(bs) - (_bitCount)) -#define DRFLAC_CACHE_L1_SELECT(bs, _bitCount) (((bs)->cache) & DRFLAC_CACHE_L1_SELECTION_MASK(_bitCount)) -#define DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, _bitCount) (DRFLAC_CACHE_L1_SELECT((bs), _bitCount) >> DRFLAC_CACHE_L1_SELECTION_SHIFT((bs), _bitCount)) -#define DRFLAC_CACHE_L2_SIZE_BYTES(bs) (sizeof((bs)->cacheL2)) -#define DRFLAC_CACHE_L2_LINE_COUNT(bs) (DRFLAC_CACHE_L2_SIZE_BYTES(bs) / sizeof((bs)->cacheL2[0])) -#define DRFLAC_CACHE_L2_LINES_REMAINING(bs) (DRFLAC_CACHE_L2_LINE_COUNT(bs) - (bs)->nextL2Line) - - -#ifndef DR_FLAC_NO_CRC -static DRFLAC_INLINE void drflac__reset_crc16(drflac_bs* bs) -{ - bs->crc16 = 0; - bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; -} - -static DRFLAC_INLINE void drflac__update_crc16(drflac_bs* bs) -{ - bs->crc16 = drflac_crc16_bytes(bs->crc16, bs->crc16Cache, DRFLAC_CACHE_L1_SIZE_BYTES(bs) - bs->crc16CacheIgnoredBytes); - bs->crc16CacheIgnoredBytes = 0; -} - -static DRFLAC_INLINE drflac_uint16 drflac__flush_crc16(drflac_bs* bs) -{ - /* We should never be flushing in a situation where we are not aligned on a byte boundary. */ - drflac_assert((DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7) == 0); - - /* The bits that were read from the L1 cache need to be accumulated. The number of bytes needing to be accumulated is determined - * by the number of bits that have been consumed. */ - if (DRFLAC_CACHE_L1_BITS_REMAINING(bs) == 0) - drflac__update_crc16(bs); - else - { - /* We only accumulate the consumed bits. */ - bs->crc16 = drflac_crc16_bytes(bs->crc16, bs->crc16Cache >> DRFLAC_CACHE_L1_BITS_REMAINING(bs), (bs->consumedBits >> 3) - bs->crc16CacheIgnoredBytes); - - /* The bits that we just accumulated should never be accumulated again. We need to keep track of how many bytes were accumulated - * so we can handle that later. */ - bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; - } - - return bs->crc16; -} -#endif - -static DRFLAC_INLINE drflac_bool32 drflac__reload_l1_cache_from_l2(drflac_bs* bs) -{ - size_t alignedL1LineCount; - size_t bytesRead; - /* Fast path. Try loading straight from L2. */ - if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) - { - bs->cache = bs->cacheL2[bs->nextL2Line++]; - return DRFLAC_TRUE; - } - - /* If we get here it means we've run out of data in the L2 cache. We'll need to fetch more from the client, if there's - * any left. */ - if (bs->unalignedByteCount > 0) - return DRFLAC_FALSE; /* If we have any unaligned bytes it means there's no more aligned bytes left in the client. */ - - bytesRead = bs->onRead(bs->pUserData, bs->cacheL2, DRFLAC_CACHE_L2_SIZE_BYTES(bs)); - - bs->nextL2Line = 0; - if (bytesRead == DRFLAC_CACHE_L2_SIZE_BYTES(bs)) - { - bs->cache = bs->cacheL2[bs->nextL2Line++]; - return DRFLAC_TRUE; - } - - - /* If we get here it means we were unable to retrieve enough data to fill the entire L2 cache. It probably - * means we've just reached the end of the file. We need to move the valid data down to the end of the buffer - * and adjust the index of the next line accordingly. Also keep in mind that the L2 cache must be aligned to - * the size of the L1 so we'll need to seek backwards by any misaligned bytes. - */ - alignedL1LineCount = bytesRead / DRFLAC_CACHE_L1_SIZE_BYTES(bs); - - /* We need to keep track of any unaligned bytes for later use. */ - bs->unalignedByteCount = bytesRead - (alignedL1LineCount * DRFLAC_CACHE_L1_SIZE_BYTES(bs)); - if (bs->unalignedByteCount > 0) - bs->unalignedCache = bs->cacheL2[alignedL1LineCount]; - - if (alignedL1LineCount > 0) - { - size_t i; - size_t offset = DRFLAC_CACHE_L2_LINE_COUNT(bs) - alignedL1LineCount; - for (i = alignedL1LineCount; i > 0; --i) - bs->cacheL2[i-1 + offset] = bs->cacheL2[i-1]; - - bs->nextL2Line = (drflac_uint32)offset; - bs->cache = bs->cacheL2[bs->nextL2Line++]; - return DRFLAC_TRUE; - } - - /* If we get into this branch it means we weren't able to load any L1-aligned data. */ - bs->nextL2Line = DRFLAC_CACHE_L2_LINE_COUNT(bs); - return DRFLAC_FALSE; -} - -static drflac_bool32 drflac__reload_cache(drflac_bs* bs) -{ - size_t bytesRead; - -#ifndef DR_FLAC_NO_CRC - drflac__update_crc16(bs); -#endif - - /* Fast path. Try just moving the next value in the L2 cache to the L1 cache. */ - if (drflac__reload_l1_cache_from_l2(bs)) - { - bs->cache = drflac__be2host__cache_line(bs->cache); - bs->consumedBits = 0; -#ifndef DR_FLAC_NO_CRC - bs->crc16Cache = bs->cache; -#endif - return DRFLAC_TRUE; - } - - /* Slow path. */ - - /* If we get here it means we have failed to load the L1 cache from the L2. Likely we've just reached the end of the stream and the last - * few bytes did not meet the alignment requirements for the L2 cache. In this case we need to fall back to a slower path and read the - * data from the unaligned cache. */ - bytesRead = bs->unalignedByteCount; - if (bytesRead == 0) - return DRFLAC_FALSE; - - drflac_assert(bytesRead < DRFLAC_CACHE_L1_SIZE_BYTES(bs)); - bs->consumedBits = (drflac_uint32)(DRFLAC_CACHE_L1_SIZE_BYTES(bs) - bytesRead) * 8; - - bs->cache = drflac__be2host__cache_line(bs->unalignedCache); - bs->cache &= DRFLAC_CACHE_L1_SELECTION_MASK(DRFLAC_CACHE_L1_SIZE_BITS(bs) - bs->consumedBits); /* <-- Make sure the consumed bits are always set to zero. Other parts of the library depend on this property. */ - bs->unalignedByteCount = 0; /* <-- At this point the unaligned bytes have been moved into the cache and we thus have no more unaligned bytes. */ - -#ifndef DR_FLAC_NO_CRC - bs->crc16Cache = bs->cache >> bs->consumedBits; - bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; -#endif - return DRFLAC_TRUE; -} - -static void drflac__reset_cache(drflac_bs* bs) -{ - bs->nextL2Line = DRFLAC_CACHE_L2_LINE_COUNT(bs); /* <-- This clears the L2 cache. */ - bs->consumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs); /* <-- This clears the L1 cache. */ - bs->cache = 0; - bs->unalignedByteCount = 0; /* <-- This clears the trailing unaligned bytes. */ - bs->unalignedCache = 0; - -#ifndef DR_FLAC_NO_CRC - bs->crc16Cache = 0; - bs->crc16CacheIgnoredBytes = 0; -#endif -} - - -static DRFLAC_INLINE drflac_bool32 drflac__read_uint32(drflac_bs* bs, unsigned int bitCount, drflac_uint32* pResultOut) -{ - drflac_assert(bs != NULL); - drflac_assert(pResultOut != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 32); - - if (bs->consumedBits == DRFLAC_CACHE_L1_SIZE_BITS(bs)) - { - if (!drflac__reload_cache(bs)) - return DRFLAC_FALSE; - } - - if (bitCount <= DRFLAC_CACHE_L1_BITS_REMAINING(bs)) - { - if (bitCount < DRFLAC_CACHE_L1_SIZE_BITS(bs)) - { - *pResultOut = (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCount); - bs->consumedBits += bitCount; - bs->cache <<= bitCount; - } else { - *pResultOut = (drflac_uint32)bs->cache; - bs->consumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs); - bs->cache = 0; - } - return DRFLAC_TRUE; - } else { - /* It straddles the cached data. It will never cover more than the next chunk. We just read the number in two parts and combine them. */ - drflac_uint32 bitCountHi = DRFLAC_CACHE_L1_BITS_REMAINING(bs); - drflac_uint32 bitCountLo = bitCount - bitCountHi; - drflac_uint32 resultHi = (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountHi); - - if (!drflac__reload_cache(bs)) - return DRFLAC_FALSE; - - *pResultOut = (drflac_uint32)(resultHi << bitCountLo) | (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountLo); - bs->consumedBits += bitCountLo; - bs->cache <<= bitCountLo; - return DRFLAC_TRUE; - } -} - -static drflac_bool32 drflac__read_int32(drflac_bs* bs, unsigned int bitCount, drflac_int32* pResult) -{ - drflac_uint32 result; - drflac_uint32 signbit; - - drflac_assert(bs != NULL); - drflac_assert(pResult != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 32); - - if (!drflac__read_uint32(bs, bitCount, &result)) - return DRFLAC_FALSE; - - signbit = ((result >> (bitCount-1)) & 0x01); - result |= (~signbit + 1) << bitCount; - - *pResult = (drflac_int32)result; - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__read_uint64(drflac_bs* bs, unsigned int bitCount, drflac_uint64* pResultOut) -{ - drflac_uint32 resultHi; - drflac_uint32 resultLo; - - drflac_assert(bitCount <= 64); - drflac_assert(bitCount > 32); - - if (!drflac__read_uint32(bs, bitCount - 32, &resultHi)) - return DRFLAC_FALSE; - - if (!drflac__read_uint32(bs, 32, &resultLo)) - return DRFLAC_FALSE; - - *pResultOut = (((drflac_uint64)resultHi) << 32) | ((drflac_uint64)resultLo); - return DRFLAC_TRUE; -} - -/* Function below is unused, but leaving it here in case I need to quickly add it again. */ -#if 0 -static drflac_bool32 drflac__read_int64(drflac_bs* bs, unsigned int bitCount, drflac_int64* pResultOut) -{ - drflac_assert(bitCount <= 64); - - drflac_uint64 result; - if (!drflac__read_uint64(bs, bitCount, &result)) - return DRFLAC_FALSE; - - drflac_uint64 signbit = ((result >> (bitCount-1)) & 0x01); - result |= (~signbit + 1) << bitCount; - - *pResultOut = (drflac_int64)result; - return DRFLAC_TRUE; -} -#endif - -static drflac_bool32 drflac__read_uint16(drflac_bs* bs, unsigned int bitCount, drflac_uint16* pResult) -{ - drflac_uint32 result; - - drflac_assert(bs != NULL); - drflac_assert(pResult != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 16); - - if (!drflac__read_uint32(bs, bitCount, &result)) - return DRFLAC_FALSE; - - *pResult = (drflac_uint16)result; - return DRFLAC_TRUE; -} - -#if 0 -static drflac_bool32 drflac__read_int16(drflac_bs* bs, unsigned int bitCount, drflac_int16* pResult) -{ - drflac_int32 result; - - drflac_assert(bs != NULL); - drflac_assert(pResult != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 16); - - if (!drflac__read_int32(bs, bitCount, &result)) - return DRFLAC_FALSE; - - *pResult = (drflac_int16)result; - return DRFLAC_TRUE; -} -#endif - -static drflac_bool32 drflac__read_uint8(drflac_bs* bs, unsigned int bitCount, drflac_uint8* pResult) -{ - drflac_uint32 result; - - drflac_assert(bs != NULL); - drflac_assert(pResult != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 8); - - if (!drflac__read_uint32(bs, bitCount, &result)) - return DRFLAC_FALSE; - - *pResult = (drflac_uint8)result; - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__read_int8(drflac_bs* bs, unsigned int bitCount, drflac_int8* pResult) -{ - drflac_int32 result; - - drflac_assert(bs != NULL); - drflac_assert(pResult != NULL); - drflac_assert(bitCount > 0); - drflac_assert(bitCount <= 8); - - if (!drflac__read_int32(bs, bitCount, &result)) - return DRFLAC_FALSE; - - *pResult = (drflac_int8)result; - return DRFLAC_TRUE; -} - - -static drflac_bool32 drflac__seek_bits(drflac_bs* bs, size_t bitsToSeek) -{ - if (bitsToSeek <= DRFLAC_CACHE_L1_BITS_REMAINING(bs)) - { - bs->consumedBits += (drflac_uint32)bitsToSeek; - bs->cache <<= bitsToSeek; - return DRFLAC_TRUE; - } - - /* It straddles the cached data. This function isn't called too frequently so I'm favouring simplicity here. */ - bitsToSeek -= DRFLAC_CACHE_L1_BITS_REMAINING(bs); - bs->consumedBits += DRFLAC_CACHE_L1_BITS_REMAINING(bs); - bs->cache = 0; - - /* Simple case. Seek in groups of the same number as bits that fit within a cache line. */ -#ifdef DRFLAC_64BIT - while (bitsToSeek >= DRFLAC_CACHE_L1_SIZE_BITS(bs)) - { - drflac_uint64 bin; - if (!drflac__read_uint64(bs, DRFLAC_CACHE_L1_SIZE_BITS(bs), &bin)) - return DRFLAC_FALSE; - bitsToSeek -= DRFLAC_CACHE_L1_SIZE_BITS(bs); - } -#else - while (bitsToSeek >= DRFLAC_CACHE_L1_SIZE_BITS(bs)) - { - drflac_uint32 bin; - if (!drflac__read_uint32(bs, DRFLAC_CACHE_L1_SIZE_BITS(bs), &bin)) - return DRFLAC_FALSE; - bitsToSeek -= DRFLAC_CACHE_L1_SIZE_BITS(bs); - } -#endif - - /* Whole leftover bytes. */ - while (bitsToSeek >= 8) - { - drflac_uint8 bin; - if (!drflac__read_uint8(bs, 8, &bin)) - return DRFLAC_FALSE; - bitsToSeek -= 8; - } - - /* Leftover bits. */ - if (bitsToSeek > 0) - { - drflac_uint8 bin; - if (!drflac__read_uint8(bs, (drflac_uint32)bitsToSeek, &bin)) - return DRFLAC_FALSE; - bitsToSeek = 0; /* <-- Necessary for the assert below. */ - } - - drflac_assert(bitsToSeek == 0); - return DRFLAC_TRUE; -} - - -/* This function moves the bit streamer to the first bit after the sync code (bit 15 of the of the frame header). It will also update the CRC-16. */ -static drflac_bool32 drflac__find_and_seek_to_next_sync_code(drflac_bs* bs) -{ - drflac_assert(bs != NULL); - - /* The sync code is always aligned to 8 bits. This is convenient for us because it means we can do byte-aligned movements. The first - * thing to do is align to the next byte. */ - if (!drflac__seek_bits(bs, DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) - return DRFLAC_FALSE; - - for (;;) - { - drflac_uint8 hi; - -#ifndef DR_FLAC_NO_CRC - drflac__reset_crc16(bs); -#endif - - if (!drflac__read_uint8(bs, 8, &hi)) - return DRFLAC_FALSE; - - if (hi == 0xFF) - { - drflac_uint8 lo; - if (!drflac__read_uint8(bs, 6, &lo)) - return DRFLAC_FALSE; - - if (lo == 0x3E) - return DRFLAC_TRUE; - - if (!drflac__seek_bits(bs, DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) - return DRFLAC_FALSE; - } - } - -#if 0 - /* Should never get here. */ - return DRFLAC_FALSE; -#endif -} - - -#if !defined(DR_FLAC_NO_SIMD) && defined(DRFLAC_HAS_LZCNT_INTRINSIC) -#define DRFLAC_IMPLEMENT_CLZ_LZCNT -#endif -#if defined(_MSC_VER) && _MSC_VER >= 1400 && (defined(DRFLAC_X64) || defined(DRFLAC_X86)) -#define DRFLAC_IMPLEMENT_CLZ_MSVC -#endif - -static DRFLAC_INLINE drflac_uint32 drflac__clz_software(drflac_cache_t x) -{ - static drflac_uint32 clz_table_4[] = { - 0, - 4, - 3, 3, - 2, 2, 2, 2, - 1, 1, 1, 1, 1, 1, 1, 1 - }; - drflac_uint32 n = clz_table_4[x >> (sizeof(x)*8 - 4)]; - - if (n == 0) - { -#ifdef DRFLAC_64BIT - if ((x & 0xFFFFFFFF00000000ULL) == 0) { n = 32; x <<= 32; } - if ((x & 0xFFFF000000000000ULL) == 0) { n += 16; x <<= 16; } - if ((x & 0xFF00000000000000ULL) == 0) { n += 8; x <<= 8; } - if ((x & 0xF000000000000000ULL) == 0) { n += 4; x <<= 4; } -#else - if ((x & 0xFFFF0000) == 0) { n = 16; x <<= 16; } - if ((x & 0xFF000000) == 0) { n += 8; x <<= 8; } - if ((x & 0xF0000000) == 0) { n += 4; x <<= 4; } -#endif - n += clz_table_4[x >> (sizeof(x)*8 - 4)]; - } - - return n - 1; -} - -#ifdef DRFLAC_IMPLEMENT_CLZ_LZCNT -static DRFLAC_INLINE drflac_bool32 drflac__is_lzcnt_supported(void) -{ - /* If the compiler itself does not support the intrinsic then we'll need to return false. */ -#ifdef DRFLAC_HAS_LZCNT_INTRINSIC - return drflac__gIsLZCNTSupported; -#else - return DRFLAC_FALSE; -#endif -} - -static DRFLAC_INLINE drflac_uint32 drflac__clz_lzcnt(drflac_cache_t x) -{ -#if defined(_MSC_VER) && !defined(__clang__) -#ifdef DRFLAC_64BIT - return (drflac_uint32)__lzcnt64(x); -#else - return (drflac_uint32)__lzcnt(x); -#endif -#else -#if defined(__GNUC__) || defined(__clang__) -#ifdef DRFLAC_64BIT - return (drflac_uint32)__builtin_clzll((unsigned long long)x); -#else - return (drflac_uint32)__builtin_clzl((unsigned long)x); -#endif -#else - /* Unsupported compiler. */ -#error "This compiler does not support the lzcnt intrinsic." -#endif -#endif -} -#endif - -#ifdef DRFLAC_IMPLEMENT_CLZ_MSVC -static DRFLAC_INLINE drflac_uint32 drflac__clz_msvc(drflac_cache_t x) -{ - drflac_uint32 n; -#ifdef DRFLAC_64BIT - _BitScanReverse64((unsigned long*)&n, x); -#else - _BitScanReverse((unsigned long*)&n, x); -#endif - return sizeof(x)*8 - n - 1; -} -#endif - -static DRFLAC_INLINE drflac_uint32 drflac__clz(drflac_cache_t x) -{ - /* This function assumes at least one bit is set. Checking for 0 needs to be done at a higher level, outside this function. */ -#ifdef DRFLAC_IMPLEMENT_CLZ_LZCNT - if (drflac__is_lzcnt_supported()) - return drflac__clz_lzcnt(x); -#endif - -#ifdef DRFLAC_IMPLEMENT_CLZ_MSVC - return drflac__clz_msvc(x); -#else - return drflac__clz_software(x); -#endif -} - -static INLINE drflac_bool32 drflac__seek_past_next_set_bit(drflac_bs* bs, unsigned int* pOffsetOut) -{ - drflac_uint32 setBitOffsetPlus1; - drflac_uint32 zeroCounter = 0; - while (bs->cache == 0) - { - zeroCounter += (drflac_uint32)DRFLAC_CACHE_L1_BITS_REMAINING(bs); - if (!drflac__reload_cache(bs)) - return DRFLAC_FALSE; - } - - setBitOffsetPlus1 = drflac__clz(bs->cache); - zeroCounter += setBitOffsetPlus1; - setBitOffsetPlus1 += 1; - - bs->consumedBits += setBitOffsetPlus1; - bs->cache <<= setBitOffsetPlus1; - - *pOffsetOut = zeroCounter + setBitOffsetPlus1 - 1; - return DRFLAC_TRUE; -} - - - -static drflac_bool32 drflac__seek_to_byte(drflac_bs* bs, drflac_uint64 offsetFromStart) -{ - drflac_assert(bs != NULL); - drflac_assert(offsetFromStart > 0); - - /* Seeking from the start is not quite as trivial as it sounds because the onSeek callback takes a signed 32-bit integer (which - * is intentional because it simplifies the implementation of the onSeek callbacks), however offsetFromStart is unsigned 64-bit. - * To resolve we just need to do an initial seek from the start, and then a series of offset seeks to make up the remainder. */ - if (offsetFromStart > 0x7FFFFFFF) - { - drflac_uint64 bytesRemaining = offsetFromStart; - if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, drflac_seek_origin_start)) - return DRFLAC_FALSE; - bytesRemaining -= 0x7FFFFFFF; - - while (bytesRemaining > 0x7FFFFFFF) - { - if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, drflac_seek_origin_current)) - return DRFLAC_FALSE; - bytesRemaining -= 0x7FFFFFFF; - } - - if (bytesRemaining > 0) - { - if (!bs->onSeek(bs->pUserData, (int)bytesRemaining, drflac_seek_origin_current)) - return DRFLAC_FALSE; - } - } - else - { - if (!bs->onSeek(bs->pUserData, (int)offsetFromStart, drflac_seek_origin_start)) - return DRFLAC_FALSE; - } - - /* The cache should be reset to force a reload of fresh data from the client. */ - drflac__reset_cache(bs); - return DRFLAC_TRUE; -} - - -static drflac_result drflac__read_utf8_coded_number(drflac_bs* bs, drflac_uint64* pNumberOut, drflac_uint8* pCRCOut) -{ - drflac_uint64 result; - int i; - int byteCount = 1; - unsigned char utf8[7] = {0}; - drflac_uint8 crc; - - drflac_assert(bs != NULL); - drflac_assert(pNumberOut != NULL); - - crc = *pCRCOut; - - if (!drflac__read_uint8(bs, 8, utf8)) - { - *pNumberOut = 0; - return DRFLAC_END_OF_STREAM; - } - crc = drflac_crc8(crc, utf8[0], 8); - - if ((utf8[0] & 0x80) == 0) - { - *pNumberOut = utf8[0]; - *pCRCOut = crc; - return DRFLAC_SUCCESS; - } - - if ((utf8[0] & 0xE0) == 0xC0) - byteCount = 2; - else if ((utf8[0] & 0xF0) == 0xE0) - byteCount = 3; - else if ((utf8[0] & 0xF8) == 0xF0) - byteCount = 4; - else if ((utf8[0] & 0xFC) == 0xF8) - byteCount = 5; - else if ((utf8[0] & 0xFE) == 0xFC) - byteCount = 6; - else if ((utf8[0] & 0xFF) == 0xFE) - byteCount = 7; - else - { - *pNumberOut = 0; - return DRFLAC_CRC_MISMATCH; /* Bad UTF-8 encoding. */ - } - - /* Read extra bytes. */ - drflac_assert(byteCount > 1); - - result = (drflac_uint64)(utf8[0] & (0xFF >> (byteCount + 1))); - for (i = 1; i < byteCount; ++i) - { - if (!drflac__read_uint8(bs, 8, utf8 + i)) - { - *pNumberOut = 0; - return DRFLAC_END_OF_STREAM; - } - crc = drflac_crc8(crc, utf8[i], 8); - - result = (result << 6) | (utf8[i] & 0x3F); - } - - *pNumberOut = result; - *pCRCOut = crc; - return DRFLAC_SUCCESS; -} - - - - -/* The next two functions are responsible for calculating the prediction. - * - * When the bits per sample is >16 we need to use 64-bit integer arithmetic because otherwise we'll run out of precision. It's - * safe to assume this will be slower on 32-bit platforms so we use a more optimal solution when the bits per sample is <=16. - */ -static DRFLAC_INLINE drflac_int32 drflac__calculate_prediction_32(drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) -{ - drflac_int32 prediction = 0; - - drflac_assert(order <= 32); - - /* 32-bit version. - - * VC++ optimizes this to a single jmp. I've not yet verified this for other compilers. */ - - switch (order) - { - case 32: prediction += coefficients[31] * pDecodedSamples[-32]; - case 31: prediction += coefficients[30] * pDecodedSamples[-31]; - case 30: prediction += coefficients[29] * pDecodedSamples[-30]; - case 29: prediction += coefficients[28] * pDecodedSamples[-29]; - case 28: prediction += coefficients[27] * pDecodedSamples[-28]; - case 27: prediction += coefficients[26] * pDecodedSamples[-27]; - case 26: prediction += coefficients[25] * pDecodedSamples[-26]; - case 25: prediction += coefficients[24] * pDecodedSamples[-25]; - case 24: prediction += coefficients[23] * pDecodedSamples[-24]; - case 23: prediction += coefficients[22] * pDecodedSamples[-23]; - case 22: prediction += coefficients[21] * pDecodedSamples[-22]; - case 21: prediction += coefficients[20] * pDecodedSamples[-21]; - case 20: prediction += coefficients[19] * pDecodedSamples[-20]; - case 19: prediction += coefficients[18] * pDecodedSamples[-19]; - case 18: prediction += coefficients[17] * pDecodedSamples[-18]; - case 17: prediction += coefficients[16] * pDecodedSamples[-17]; - case 16: prediction += coefficients[15] * pDecodedSamples[-16]; - case 15: prediction += coefficients[14] * pDecodedSamples[-15]; - case 14: prediction += coefficients[13] * pDecodedSamples[-14]; - case 13: prediction += coefficients[12] * pDecodedSamples[-13]; - case 12: prediction += coefficients[11] * pDecodedSamples[-12]; - case 11: prediction += coefficients[10] * pDecodedSamples[-11]; - case 10: prediction += coefficients[ 9] * pDecodedSamples[-10]; - case 9: prediction += coefficients[ 8] * pDecodedSamples[- 9]; - case 8: prediction += coefficients[ 7] * pDecodedSamples[- 8]; - case 7: prediction += coefficients[ 6] * pDecodedSamples[- 7]; - case 6: prediction += coefficients[ 5] * pDecodedSamples[- 6]; - case 5: prediction += coefficients[ 4] * pDecodedSamples[- 5]; - case 4: prediction += coefficients[ 3] * pDecodedSamples[- 4]; - case 3: prediction += coefficients[ 2] * pDecodedSamples[- 3]; - case 2: prediction += coefficients[ 1] * pDecodedSamples[- 2]; - case 1: prediction += coefficients[ 0] * pDecodedSamples[- 1]; - } - - return (drflac_int32)(prediction >> shift); -} - -static DRFLAC_INLINE drflac_int32 drflac__calculate_prediction_64(drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) -{ - drflac_int64 prediction; - drflac_assert(order <= 32); - - /* 64-bit version. - - * This method is faster on the 32-bit build when compiling with VC++. See note below. - */ - -#ifndef DRFLAC_64BIT - if (order == 8) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; - } - else if (order == 7) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - } - else if (order == 3) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - } - else if (order == 6) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - } - else if (order == 5) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - } - else if (order == 4) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - } - else if (order == 12) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; - prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; - prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; - prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; - prediction += coefficients[11] * (drflac_int64)pDecodedSamples[-12]; - } - else if (order == 2) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - } - else if (order == 1) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - } - else if (order == 10) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; - prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; - prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; - } - else if (order == 9) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; - prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; - } - else if (order == 11) - { - prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; - prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; - prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; - prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; - prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; - prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; - prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; - prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; - prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; - prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; - prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; - } - else - { - int j; - prediction = 0; - for (j = 0; j < (int)order; ++j) - prediction += coefficients[j] * (drflac_int64)pDecodedSamples[-j-1]; - } -#endif - - /* VC++ optimizes this to a single jmp instruction, but only the 64-bit build. The 32-bit build generates less efficient code for some - * reason. The ugly version above is faster so we'll just switch between the two depending on the target platform. */ -#ifdef DRFLAC_64BIT - prediction = 0; - - switch (order) - { - case 32: prediction += coefficients[31] * (drflac_int64)pDecodedSamples[-32]; - case 31: prediction += coefficients[30] * (drflac_int64)pDecodedSamples[-31]; - case 30: prediction += coefficients[29] * (drflac_int64)pDecodedSamples[-30]; - case 29: prediction += coefficients[28] * (drflac_int64)pDecodedSamples[-29]; - case 28: prediction += coefficients[27] * (drflac_int64)pDecodedSamples[-28]; - case 27: prediction += coefficients[26] * (drflac_int64)pDecodedSamples[-27]; - case 26: prediction += coefficients[25] * (drflac_int64)pDecodedSamples[-26]; - case 25: prediction += coefficients[24] * (drflac_int64)pDecodedSamples[-25]; - case 24: prediction += coefficients[23] * (drflac_int64)pDecodedSamples[-24]; - case 23: prediction += coefficients[22] * (drflac_int64)pDecodedSamples[-23]; - case 22: prediction += coefficients[21] * (drflac_int64)pDecodedSamples[-22]; - case 21: prediction += coefficients[20] * (drflac_int64)pDecodedSamples[-21]; - case 20: prediction += coefficients[19] * (drflac_int64)pDecodedSamples[-20]; - case 19: prediction += coefficients[18] * (drflac_int64)pDecodedSamples[-19]; - case 18: prediction += coefficients[17] * (drflac_int64)pDecodedSamples[-18]; - case 17: prediction += coefficients[16] * (drflac_int64)pDecodedSamples[-17]; - case 16: prediction += coefficients[15] * (drflac_int64)pDecodedSamples[-16]; - case 15: prediction += coefficients[14] * (drflac_int64)pDecodedSamples[-15]; - case 14: prediction += coefficients[13] * (drflac_int64)pDecodedSamples[-14]; - case 13: prediction += coefficients[12] * (drflac_int64)pDecodedSamples[-13]; - case 12: prediction += coefficients[11] * (drflac_int64)pDecodedSamples[-12]; - case 11: prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; - case 10: prediction += coefficients[ 9] * (drflac_int64)pDecodedSamples[-10]; - case 9: prediction += coefficients[ 8] * (drflac_int64)pDecodedSamples[- 9]; - case 8: prediction += coefficients[ 7] * (drflac_int64)pDecodedSamples[- 8]; - case 7: prediction += coefficients[ 6] * (drflac_int64)pDecodedSamples[- 7]; - case 6: prediction += coefficients[ 5] * (drflac_int64)pDecodedSamples[- 6]; - case 5: prediction += coefficients[ 4] * (drflac_int64)pDecodedSamples[- 5]; - case 4: prediction += coefficients[ 3] * (drflac_int64)pDecodedSamples[- 4]; - case 3: prediction += coefficients[ 2] * (drflac_int64)pDecodedSamples[- 3]; - case 2: prediction += coefficients[ 1] * (drflac_int64)pDecodedSamples[- 2]; - case 1: prediction += coefficients[ 0] * (drflac_int64)pDecodedSamples[- 1]; - } -#endif - - return (drflac_int32)(prediction >> shift); -} - -#if 0 -/* Reference implementation for reading and decoding samples with residual. This is intentionally left unoptimized for the - * sake of readability and should only be used as a reference. */ -static drflac_bool32 drflac__decode_samples_with_residual__rice__reference(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) -{ - drflac_assert(bs != NULL); - drflac_assert(count > 0); - drflac_assert(pSamplesOut != NULL); - - for (drflac_uint32 i = 0; i < count; ++i) - { - drflac_uint32 zeroCounter = 0; - for (;;) - { - drflac_uint8 bit; - if (!drflac__read_uint8(bs, 1, &bit)) - return DRFLAC_FALSE; - - if (bit == 0) - zeroCounter += 1; - else - break; - } - - drflac_uint32 decodedRice; - if (riceParam > 0) - { - if (!drflac__read_uint32(bs, riceParam, &decodedRice)) - return DRFLAC_FALSE; - } - else - decodedRice = 0; - - decodedRice |= (zeroCounter << riceParam); - if ((decodedRice & 0x01)) - decodedRice = ~(decodedRice >> 1); - else - decodedRice = (decodedRice >> 1); - - - if (bitsPerSample > 16) - pSamplesOut[i] = decodedRice + drflac__calculate_prediction_64(order, shift, coefficients, pSamplesOut + i); - else - pSamplesOut[i] = decodedRice + drflac__calculate_prediction_32(order, shift, coefficients, pSamplesOut + i); - } - - return DRFLAC_TRUE; -} -#endif - -#if 0 -static drflac_bool32 drflac__read_rice_parts__reference(drflac_bs* bs, drflac_uint8 riceParam, drflac_uint32* pZeroCounterOut, drflac_uint32* pRiceParamPartOut) -{ - drflac_uint32 zeroCounter = 0; - for (;;) { - drflac_uint8 bit; - if (!drflac__read_uint8(bs, 1, &bit)) - return DRFLAC_FALSE; - - if (bit == 0) - zeroCounter += 1; - else - break; - } - - drflac_uint32 decodedRice; - if (riceParam > 0) - { - if (!drflac__read_uint32(bs, riceParam, &decodedRice)) - return DRFLAC_FALSE; - } - else - decodedRice = 0; - - *pZeroCounterOut = zeroCounter; - *pRiceParamPartOut = decodedRice; - return DRFLAC_TRUE; -} -#endif - -static DRFLAC_INLINE drflac_bool32 drflac__read_rice_parts(drflac_bs* bs, drflac_uint8 riceParam, drflac_uint32* pZeroCounterOut, drflac_uint32* pRiceParamPartOut) -{ - drflac_uint32 riceLength; - drflac_uint32 riceParamPart = 0; - drflac_uint32 setBitOffsetPlus1; - drflac_cache_t riceParamMask = DRFLAC_CACHE_L1_SELECTION_MASK(riceParam); - drflac_cache_t resultHiShift = DRFLAC_CACHE_L1_SIZE_BITS(bs) - riceParam; - drflac_uint32 zeroCounter = 0; - while (bs->cache == 0) { - zeroCounter += (drflac_uint32)DRFLAC_CACHE_L1_BITS_REMAINING(bs); - if (!drflac__reload_cache(bs)) - return DRFLAC_FALSE; - } - - setBitOffsetPlus1 = drflac__clz(bs->cache); - zeroCounter += setBitOffsetPlus1; - setBitOffsetPlus1 += 1; - - riceLength = setBitOffsetPlus1 + riceParam; - - if (riceLength < DRFLAC_CACHE_L1_BITS_REMAINING(bs)) - { - riceParamPart = (drflac_uint32)((bs->cache & (riceParamMask >> setBitOffsetPlus1)) >> (DRFLAC_CACHE_L1_SIZE_BITS(bs) - riceLength)); - - bs->consumedBits += riceLength; - bs->cache <<= riceLength; - } - else - { - drflac_uint32 bitCountLo; - drflac_cache_t resultHi; - - bs->consumedBits += riceLength; - if (setBitOffsetPlus1 < DRFLAC_CACHE_L1_SIZE_BITS(bs)) - bs->cache <<= setBitOffsetPlus1; - - /* It straddles the cached data. It will never cover more than the next chunk. We just read the number in two parts and combine them. */ - bitCountLo = bs->consumedBits - DRFLAC_CACHE_L1_SIZE_BITS(bs); - resultHi = bs->cache & riceParamMask; /* <-- This mask is OK because all bits after the first bits are always zero. */ - - if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { -#ifndef DR_FLAC_NO_CRC - drflac__update_crc16(bs); -#endif - bs->cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); - bs->consumedBits = 0; -#ifndef DR_FLAC_NO_CRC - bs->crc16Cache = bs->cache; -#endif - } else { - /* Slow path. We need to fetch more data from the client. */ - if (!drflac__reload_cache(bs)) { - return DRFLAC_FALSE; - } - } - - riceParamPart = (drflac_uint32)((resultHi >> resultHiShift) | DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountLo)); - - bs->consumedBits += bitCountLo; - bs->cache <<= bitCountLo; - } - - *pZeroCounterOut = zeroCounter; - *pRiceParamPartOut = riceParamPart; - return DRFLAC_TRUE; -} - - -static drflac_bool32 drflac__decode_samples_with_residual__rice__simple(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) -{ - drflac_uint32 zeroCountPart = 0; - drflac_uint32 riceParamPart = 0; - drflac_uint32 i = 0; - - drflac_assert(bs != NULL); - drflac_assert(count > 0); - drflac_assert(pSamplesOut != NULL); - - while (i < count) - { - static drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; - /* Rice extraction. */ - if (!drflac__read_rice_parts(bs, riceParam, &zeroCountPart, &riceParamPart)) - return DRFLAC_FALSE; - - /* Rice reconstruction. */ - - riceParamPart |= (zeroCountPart << riceParam); - riceParamPart = (riceParamPart >> 1) ^ t[riceParamPart & 0x01]; -#if 0 - riceParamPart = (riceParamPart >> 1) ^ (~(riceParamPart & 0x01) + 1); -#endif - - /* Sample reconstruction. */ - if (bitsPerSample > 16) - { - pSamplesOut[i] = riceParamPart + drflac__calculate_prediction_64(order, shift, coefficients, pSamplesOut + i); - } - else - { - pSamplesOut[i] = riceParamPart + drflac__calculate_prediction_32(order, shift, coefficients, pSamplesOut + i); - } - - i += 1; - } - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_samples_with_residual__rice(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) -{ -#if 0 - return drflac__decode_samples_with_residual__rice__reference(bs, bitsPerSample, count, riceParam, order, shift, coefficients, pSamplesOut); -#else - return drflac__decode_samples_with_residual__rice__simple(bs, bitsPerSample, count, riceParam, order, shift, coefficients, pSamplesOut); -#endif -} - -/* Reads and seeks past a string of residual values as Rice codes. The decoder should be sitting on the first bit of the Rice codes. */ -static drflac_bool32 drflac__read_and_seek_residual__rice(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam) -{ - drflac_uint32 i; - drflac_assert(bs != NULL); - drflac_assert(count > 0); - - for (i = 0; i < count; ++i) - { - drflac_uint32 zeroCountPart; - drflac_uint32 riceParamPart; - if (!drflac__read_rice_parts(bs, riceParam, &zeroCountPart, &riceParamPart)) - return DRFLAC_FALSE; - } - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_samples_with_residual__unencoded(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 unencodedBitsPerSample, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) -{ - unsigned int i; - drflac_assert(bs != NULL); - drflac_assert(count > 0); - drflac_assert(unencodedBitsPerSample > 0 && unencodedBitsPerSample <= 32); - drflac_assert(pSamplesOut != NULL); - - for (i = 0; i < count; ++i) - { - if (!drflac__read_int32(bs, unencodedBitsPerSample, pSamplesOut + i)) - return DRFLAC_FALSE; - - if (bitsPerSample > 16) - { - pSamplesOut[i] += drflac__calculate_prediction_64(order, shift, coefficients, pSamplesOut + i); - } - else - { - pSamplesOut[i] += drflac__calculate_prediction_32(order, shift, coefficients, pSamplesOut + i); - } - } - - return DRFLAC_TRUE; -} - - -/* Reads and decodes the residual for the sub-frame the decoder is currently sitting on. This function should be called - * when the decoder is sitting at the very start of the RESIDUAL block. The first residuals will be ignored. The - * and parameters are used to determine how many residual values need to be decoded. */ -static drflac_bool32 drflac__decode_samples_with_residual(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 blockSize, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) -{ - drflac_uint8 partitionOrder; - drflac_uint8 residualMethod; - drflac_uint32 samplesInPartition; - drflac_uint32 partitionsRemaining; - - drflac_assert(bs != NULL); - drflac_assert(blockSize != 0); - drflac_assert(pDecodedSamples != NULL); /* <-- Should we allow NULL, in which case we just seek past the residual rather than do a full decode? */ - - if (!drflac__read_uint8(bs, 2, &residualMethod)) - return DRFLAC_FALSE; - - if (residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) - return DRFLAC_FALSE; /* Unknown or unsupported residual coding method. */ - - /* Ignore the first values. */ - pDecodedSamples += order; - - - if (!drflac__read_uint8(bs, 4, &partitionOrder)) - return DRFLAC_FALSE; - - /* From the FLAC spec: - * The Rice partition order in a Rice-coded residual section must be less than or equal to 8. */ - if (partitionOrder > 8) - return DRFLAC_FALSE; - - /* Validation check. */ - if ((blockSize / (1 << partitionOrder)) <= order) - return DRFLAC_FALSE; - - samplesInPartition = (blockSize / (1 << partitionOrder)) - order; - partitionsRemaining = (1 << partitionOrder); - - for (;;) - { - drflac_uint8 riceParam = 0; - if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) - { - if (!drflac__read_uint8(bs, 4, &riceParam)) - return DRFLAC_FALSE; - if (riceParam == 16) - riceParam = 0xFF; - } - else if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) - { - if (!drflac__read_uint8(bs, 5, &riceParam)) - return DRFLAC_FALSE; - if (riceParam == 32) - riceParam = 0xFF; - } - - if (riceParam != 0xFF) - { - if (!drflac__decode_samples_with_residual__rice(bs, bitsPerSample, samplesInPartition, riceParam, order, shift, coefficients, pDecodedSamples)) - return DRFLAC_FALSE; - } - else - { - unsigned char unencodedBitsPerSample = 0; - if (!drflac__read_uint8(bs, 5, &unencodedBitsPerSample)) - return DRFLAC_FALSE; - - if (!drflac__decode_samples_with_residual__unencoded(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, order, shift, coefficients, pDecodedSamples)) - return DRFLAC_FALSE; - } - - pDecodedSamples += samplesInPartition; - - - if (partitionsRemaining == 1) - break; - - partitionsRemaining -= 1; - - if (partitionOrder != 0) - samplesInPartition = blockSize / (1 << partitionOrder); - } - - return DRFLAC_TRUE; -} - -/* Reads and seeks past the residual for the sub-frame the decoder is currently sitting on. This function should be called - * when the decoder is sitting at the very start of the RESIDUAL block. The first residuals will be set to 0. The - * and parameters are used to determine how many residual values need to be decoded. */ -static drflac_bool32 drflac__read_and_seek_residual(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 order) -{ - drflac_uint32 partitionsRemaining; - drflac_uint32 samplesInPartition; - drflac_uint8 partitionOrder; - drflac_uint8 residualMethod; - - drflac_assert(bs != NULL); - drflac_assert(blockSize != 0); - - if (!drflac__read_uint8(bs, 2, &residualMethod)) - return DRFLAC_FALSE; - - if (residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) - return DRFLAC_FALSE; /* Unknown or unsupported residual coding method. */ - - if (!drflac__read_uint8(bs, 4, &partitionOrder)) - return DRFLAC_FALSE; - - samplesInPartition = (blockSize / (1 << partitionOrder)) - order; - partitionsRemaining = (1 << partitionOrder); - for (;;) - { - drflac_uint8 riceParam = 0; - if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) - { - if (!drflac__read_uint8(bs, 4, &riceParam)) - return DRFLAC_FALSE; - if (riceParam == 16) - riceParam = 0xFF; - } - else if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { - if (!drflac__read_uint8(bs, 5, &riceParam)) - return DRFLAC_FALSE; - if (riceParam == 32) - riceParam = 0xFF; - } - - if (riceParam != 0xFF) - { - if (!drflac__read_and_seek_residual__rice(bs, samplesInPartition, riceParam)) - return DRFLAC_FALSE; - } - else - { - unsigned char unencodedBitsPerSample = 0; - if (!drflac__read_uint8(bs, 5, &unencodedBitsPerSample)) - return DRFLAC_FALSE; - - if (!drflac__seek_bits(bs, unencodedBitsPerSample * samplesInPartition)) - return DRFLAC_FALSE; - } - - if (partitionsRemaining == 1) - break; - - partitionsRemaining -= 1; - samplesInPartition = blockSize / (1 << partitionOrder); - } - - return DRFLAC_TRUE; -} - - -static drflac_bool32 drflac__decode_samples__constant(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 bitsPerSample, drflac_int32* pDecodedSamples) -{ - drflac_uint32 i; - /* Only a single sample needs to be decoded here. */ - drflac_int32 sample; - if (!drflac__read_int32(bs, bitsPerSample, &sample)) - return DRFLAC_FALSE; - - /* We don't really need to expand this, but it does simplify the process of reading samples. If this becomes a performance issue (unlikely) - * we'll want to look at a more efficient way. */ - for (i = 0; i < blockSize; ++i) - pDecodedSamples[i] = sample; - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_samples__verbatim(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 bitsPerSample, drflac_int32* pDecodedSamples) -{ - drflac_uint32 i; - for (i = 0; i < blockSize; ++i) - { - drflac_int32 sample; - if (!drflac__read_int32(bs, bitsPerSample, &sample)) - return DRFLAC_FALSE; - - pDecodedSamples[i] = sample; - } - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_samples__fixed(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 bitsPerSample, drflac_uint8 lpcOrder, drflac_int32* pDecodedSamples) -{ - drflac_uint32 i; - drflac_int32 lpcCoefficientsTable[5][4] = { - {0, 0, 0, 0}, - {1, 0, 0, 0}, - {2, -1, 0, 0}, - {3, -3, 1, 0}, - {4, -6, 4, -1} - }; - - /* Warm up samples and coefficients. */ - for (i = 0; i < lpcOrder; ++i) - { - drflac_int32 sample; - if (!drflac__read_int32(bs, bitsPerSample, &sample)) - return DRFLAC_FALSE; - - pDecodedSamples[i] = sample; - } - - - if (!drflac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, 0, lpcCoefficientsTable[lpcOrder], pDecodedSamples)) - return DRFLAC_FALSE; - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_samples__lpc(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 bitsPerSample, drflac_uint8 lpcOrder, drflac_int32* pDecodedSamples) -{ - drflac_uint8 i; - drflac_uint8 lpcPrecision; - drflac_int8 lpcShift; - drflac_int32 coefficients[32]; - - /* Warm up samples. */ - for (i = 0; i < lpcOrder; ++i) - { - drflac_int32 sample; - if (!drflac__read_int32(bs, bitsPerSample, &sample)) - return DRFLAC_FALSE; - - pDecodedSamples[i] = sample; - } - - if (!drflac__read_uint8(bs, 4, &lpcPrecision)) - return DRFLAC_FALSE; - if (lpcPrecision == 15) - return DRFLAC_FALSE; /* Invalid. */ - lpcPrecision += 1; - - - if (!drflac__read_int8(bs, 5, &lpcShift)) - return DRFLAC_FALSE; - - for (i = 0; i < lpcOrder; ++i) - { - if (!drflac__read_int32(bs, lpcPrecision, coefficients + i)) - return DRFLAC_FALSE; - } - - if (!drflac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, lpcShift, coefficients, pDecodedSamples)) - return DRFLAC_FALSE; - - return DRFLAC_TRUE; -} - - -static drflac_bool32 drflac__read_next_frame_header(drflac_bs* bs, drflac_uint8 streaminfoBitsPerSample, drflac_frame_header* header) -{ - const drflac_uint32 sampleRateTable[12] = {0, 88200, 176400, 192000, 8000, 16000, 22050, 24000, 32000, 44100, 48000, 96000}; - const drflac_uint8 bitsPerSampleTable[8] = {0, 8, 12, (drflac_uint8)-1, 16, 20, 24, (drflac_uint8)-1}; /* -1 = reserved. */ - - drflac_assert(bs != NULL); - drflac_assert(header != NULL); - - /* Keep looping until we find a valid sync code. */ - for (;;) - { - drflac_bool32 isVariableBlockSize = false; - drflac_uint8 blockSize = 0; - drflac_uint8 blockingStrategy = 0; - drflac_uint8 crc8 = 0xCE; /* 0xCE = drflac_crc8(0, 0x3FFE, 14); */ - drflac_uint8 reserved = 0; - drflac_uint8 sampleRate = 0; - drflac_uint8 channelAssignment = 0; - drflac_uint8 bitsPerSample = 0; - - if (!drflac__find_and_seek_to_next_sync_code(bs)) - return DRFLAC_FALSE; - - if (!drflac__read_uint8(bs, 1, &reserved)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, reserved, 1); - - if (!drflac__read_uint8(bs, 1, &blockingStrategy)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, blockingStrategy, 1); - - if (!drflac__read_uint8(bs, 4, &blockSize)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, blockSize, 4); - - if (!drflac__read_uint8(bs, 4, &sampleRate)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, sampleRate, 4); - - if (!drflac__read_uint8(bs, 4, &channelAssignment)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, channelAssignment, 4); - - if (!drflac__read_uint8(bs, 3, &bitsPerSample)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, bitsPerSample, 3); - - if (!drflac__read_uint8(bs, 1, &reserved)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, reserved, 1); - - isVariableBlockSize = blockingStrategy == 1; - - if (isVariableBlockSize) - { - drflac_uint64 sampleNumber; - drflac_result result = drflac__read_utf8_coded_number(bs, &sampleNumber, &crc8); - if (result != DRFLAC_SUCCESS) - { - if (result == DRFLAC_END_OF_STREAM) - return DRFLAC_FALSE; - else - continue; - } - header->frameNumber = 0; - header->sampleNumber = sampleNumber; - } - else - { - drflac_uint64 frameNumber = 0; - drflac_result result = drflac__read_utf8_coded_number(bs, &frameNumber, &crc8); - if (result != DRFLAC_SUCCESS) - { - if (result == DRFLAC_END_OF_STREAM) - return DRFLAC_FALSE; - else - continue; - } - header->frameNumber = (drflac_uint32)frameNumber; /* <-- Safe cast. */ - header->sampleNumber = 0; - } - - - if (blockSize == 1) - header->blockSize = 192; - else if (blockSize >= 2 && blockSize <= 5) - header->blockSize = 576 * (1 << (blockSize - 2)); - else if (blockSize == 6) - { - if (!drflac__read_uint16(bs, 8, &header->blockSize)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, header->blockSize, 8); - header->blockSize += 1; - } - else if (blockSize == 7) - { - if (!drflac__read_uint16(bs, 16, &header->blockSize)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, header->blockSize, 16); - header->blockSize += 1; - } - else - header->blockSize = 256 * (1 << (blockSize - 8)); - - if (sampleRate <= 11) - header->sampleRate = sampleRateTable[sampleRate]; - else if (sampleRate == 12) - { - if (!drflac__read_uint32(bs, 8, &header->sampleRate)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, header->sampleRate, 8); - header->sampleRate *= 1000; - } - else if (sampleRate == 13) - { - if (!drflac__read_uint32(bs, 16, &header->sampleRate)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, header->sampleRate, 16); - } - else if (sampleRate == 14) - { - if (!drflac__read_uint32(bs, 16, &header->sampleRate)) - return DRFLAC_FALSE; - crc8 = drflac_crc8(crc8, header->sampleRate, 16); - header->sampleRate *= 10; - } - else - continue; /* Invalid. Assume an invalid block. */ - - header->channelAssignment = channelAssignment; - - header->bitsPerSample = bitsPerSampleTable[bitsPerSample]; - if (header->bitsPerSample == 0) - header->bitsPerSample = streaminfoBitsPerSample; - - if (!drflac__read_uint8(bs, 8, &header->crc8)) - return DRFLAC_FALSE; - -#ifndef DR_FLAC_NO_CRC - if (header->crc8 != crc8) - continue; /* CRC mismatch. Loop back to the top and find the next sync code. */ -#endif - return DRFLAC_TRUE; - } -} - -static drflac_bool32 drflac__read_subframe_header(drflac_bs* bs, drflac_subframe* pSubframe) -{ - int type; - drflac_uint8 header; - if (!drflac__read_uint8(bs, 8, &header)) - return DRFLAC_FALSE; - - /* First bit should always be 0. */ - if ((header & 0x80) != 0) - return DRFLAC_FALSE; - - type = (header & 0x7E) >> 1; - if (type == 0) - pSubframe->subframeType = DRFLAC_SUBFRAME_CONSTANT; - else if (type == 1) - pSubframe->subframeType = DRFLAC_SUBFRAME_VERBATIM; - else - { - if ((type & 0x20) != 0) { - pSubframe->subframeType = DRFLAC_SUBFRAME_LPC; - pSubframe->lpcOrder = (type & 0x1F) + 1; - } else if ((type & 0x08) != 0) { - pSubframe->subframeType = DRFLAC_SUBFRAME_FIXED; - pSubframe->lpcOrder = (type & 0x07); - if (pSubframe->lpcOrder > 4) { - pSubframe->subframeType = DRFLAC_SUBFRAME_RESERVED; - pSubframe->lpcOrder = 0; - } - } else { - pSubframe->subframeType = DRFLAC_SUBFRAME_RESERVED; - } - } - - if (pSubframe->subframeType == DRFLAC_SUBFRAME_RESERVED) - return DRFLAC_FALSE; - - /* Wasted bits per sample. */ - pSubframe->wastedBitsPerSample = 0; - if ((header & 0x01) == 1) - { - unsigned int wastedBitsPerSample = 0; - if (!drflac__seek_past_next_set_bit(bs, &wastedBitsPerSample)) - return DRFLAC_FALSE; - pSubframe->wastedBitsPerSample = (unsigned char)wastedBitsPerSample + 1; - } - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__decode_subframe(drflac_bs* bs, drflac_frame* frame, int subframeIndex, drflac_int32* pDecodedSamplesOut) -{ - drflac_subframe* pSubframe = NULL; - drflac_assert(bs != NULL); - drflac_assert(frame != NULL); - - pSubframe = frame->subframes + subframeIndex; - if (!drflac__read_subframe_header(bs, pSubframe)) - return DRFLAC_FALSE; - - /* Side channels require an extra bit per sample. Took a while to figure that one out... */ - pSubframe->bitsPerSample = frame->header.bitsPerSample; - if ((frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) { - pSubframe->bitsPerSample += 1; - } else if (frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) { - pSubframe->bitsPerSample += 1; - } - - /* Need to handle wasted bits per sample. */ - pSubframe->bitsPerSample -= pSubframe->wastedBitsPerSample; - pSubframe->pDecodedSamples = pDecodedSamplesOut; - - switch (pSubframe->subframeType) - { - case DRFLAC_SUBFRAME_CONSTANT: - { - drflac__decode_samples__constant(bs, frame->header.blockSize, pSubframe->bitsPerSample, pSubframe->pDecodedSamples); - } break; - - case DRFLAC_SUBFRAME_VERBATIM: - { - drflac__decode_samples__verbatim(bs, frame->header.blockSize, pSubframe->bitsPerSample, pSubframe->pDecodedSamples); - } break; - - case DRFLAC_SUBFRAME_FIXED: - { - drflac__decode_samples__fixed(bs, frame->header.blockSize, pSubframe->bitsPerSample, pSubframe->lpcOrder, pSubframe->pDecodedSamples); - } break; - - case DRFLAC_SUBFRAME_LPC: - { - drflac__decode_samples__lpc(bs, frame->header.blockSize, pSubframe->bitsPerSample, pSubframe->lpcOrder, pSubframe->pDecodedSamples); - } break; - - default: return DRFLAC_FALSE; - } - - return DRFLAC_TRUE; -} - -static drflac_bool32 drflac__seek_subframe(drflac_bs* bs, drflac_frame* frame, int subframeIndex) -{ - drflac_subframe *pSubframe; - - drflac_assert(bs != NULL); - drflac_assert(frame != NULL); - - pSubframe = frame->subframes + subframeIndex; - if (!drflac__read_subframe_header(bs, pSubframe)) - return DRFLAC_FALSE; - - /* Side channels require an extra bit per sample. Took a while to figure that one out... */ - pSubframe->bitsPerSample = frame->header.bitsPerSample; - if ((frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) { - pSubframe->bitsPerSample += 1; - } else if (frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) { - pSubframe->bitsPerSample += 1; - } - - /* Need to handle wasted bits per sample. */ - pSubframe->bitsPerSample -= pSubframe->wastedBitsPerSample; - pSubframe->pDecodedSamples = NULL; - - switch (pSubframe->subframeType) - { - case DRFLAC_SUBFRAME_CONSTANT: - { - if (!drflac__seek_bits(bs, pSubframe->bitsPerSample)) - return DRFLAC_FALSE; - } break; - - case DRFLAC_SUBFRAME_VERBATIM: - { - unsigned int bitsToSeek = frame->header.blockSize * pSubframe->bitsPerSample; - if (!drflac__seek_bits(bs, bitsToSeek)) - return DRFLAC_FALSE; - } break; - - case DRFLAC_SUBFRAME_FIXED: - { - unsigned int bitsToSeek = pSubframe->lpcOrder * pSubframe->bitsPerSample; - if (!drflac__seek_bits(bs, bitsToSeek)) - return DRFLAC_FALSE; - - if (!drflac__read_and_seek_residual(bs, frame->header.blockSize, pSubframe->lpcOrder)) - return DRFLAC_FALSE; - } break; - - case DRFLAC_SUBFRAME_LPC: - { - unsigned char lpcPrecision; - unsigned int bitsToSeek = pSubframe->lpcOrder * pSubframe->bitsPerSample; - if (!drflac__seek_bits(bs, bitsToSeek)) - return DRFLAC_FALSE; - - if (!drflac__read_uint8(bs, 4, &lpcPrecision)) - return DRFLAC_FALSE; - if (lpcPrecision == 15) - return DRFLAC_FALSE; /* Invalid. */ - lpcPrecision += 1; - - - bitsToSeek = (pSubframe->lpcOrder * lpcPrecision) + 5; /* +5 for shift. */ - if (!drflac__seek_bits(bs, bitsToSeek)) - return DRFLAC_FALSE; - - if (!drflac__read_and_seek_residual(bs, frame->header.blockSize, pSubframe->lpcOrder)) - return DRFLAC_FALSE; - } break; - - default: return DRFLAC_FALSE; - } - - return DRFLAC_TRUE; -} - - -static DRFLAC_INLINE drflac_uint8 drflac__get_channel_count_from_channel_assignment(drflac_int8 channelAssignment) -{ - drflac_uint8 lookup[] = {1, 2, 3, 4, 5, 6, 7, 8, 2, 2, 2}; - - drflac_assert(channelAssignment <= 10); - return lookup[channelAssignment]; -} - -static drflac_result drflac__decode_frame(drflac* pFlac) -{ - drflac_uint16 actualCRC16; - drflac_uint8 paddingSizeInBits; - drflac_uint16 desiredCRC16; - int i, channelCount; - - /* This function should be called while the stream is sitting on the first byte after the frame header. */ - drflac_zero_memory(pFlac->currentFrame.subframes, sizeof(pFlac->currentFrame.subframes)); - - /* The frame block size must never be larger than the maximum block size defined by the FLAC stream. */ - if (pFlac->currentFrame.header.blockSize > pFlac->maxBlockSize) - return DRFLAC_ERROR; - - /* The number of channels in the frame must match the channel count from the STREAMINFO block. */ - channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFrame.header.channelAssignment); - if (channelCount != (int)pFlac->channels) - return DRFLAC_ERROR; - - for (i = 0; i < channelCount; ++i) - { - if (!drflac__decode_subframe(&pFlac->bs, &pFlac->currentFrame, i, pFlac->pDecodedSamples + (pFlac->currentFrame.header.blockSize * i))) - return DRFLAC_ERROR; - } - - paddingSizeInBits = DRFLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7; - if (paddingSizeInBits > 0) - { - drflac_uint8 padding = 0; - if (!drflac__read_uint8(&pFlac->bs, paddingSizeInBits, &padding)) - return DRFLAC_END_OF_STREAM; - } - -#ifndef DR_FLAC_NO_CRC - actualCRC16 = drflac__flush_crc16(&pFlac->bs); -#endif - if (!drflac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) - return DRFLAC_END_OF_STREAM; - -#ifndef DR_FLAC_NO_CRC - if (actualCRC16 != desiredCRC16) - return DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ -#endif - - pFlac->currentFrame.samplesRemaining = pFlac->currentFrame.header.blockSize * channelCount; - - return DRFLAC_SUCCESS; -} - -static drflac_result drflac__seek_frame(drflac* pFlac) -{ -#ifndef DR_FLAC_NO_CRC - drflac_uint16 actualCRC16; -#endif - drflac_uint16 desiredCRC16; - int i; - int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFrame.header.channelAssignment); - - for (i = 0; i < channelCount; ++i) - { - if (!drflac__seek_subframe(&pFlac->bs, &pFlac->currentFrame, i)) - return DRFLAC_ERROR; - } - - /* Padding. */ - if (!drflac__seek_bits(&pFlac->bs, DRFLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7)) - return DRFLAC_ERROR; - - /* CRC. */ -#ifndef DR_FLAC_NO_CRC - actualCRC16 = drflac__flush_crc16(&pFlac->bs); -#endif - if (!drflac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) - return DRFLAC_END_OF_STREAM; - -#ifndef DR_FLAC_NO_CRC - if (actualCRC16 != desiredCRC16) - return DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ -#endif - - return DRFLAC_SUCCESS; -} - -static drflac_bool32 drflac__read_and_decode_next_frame(drflac* pFlac) -{ - drflac_assert(pFlac != NULL); - - for (;;) - { - drflac_result result; - - if (!drflac__read_next_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFrame.header)) - return DRFLAC_FALSE; - - result = drflac__decode_frame(pFlac); - if (result != DRFLAC_SUCCESS) - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Skip to the next frame. */ - return DRFLAC_FALSE; - } - - return DRFLAC_TRUE; - } -} - - -static void drflac__get_current_frame_sample_range(drflac* pFlac, drflac_uint64* pFirstSampleInFrameOut, drflac_uint64* pLastSampleInFrameOut) -{ - unsigned int channelCount; - drflac_uint64 firstSampleInFrame, lastSampleInFrame; - - drflac_assert(pFlac != NULL); - - channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFrame.header.channelAssignment); - - firstSampleInFrame = pFlac->currentFrame.header.sampleNumber; - if (firstSampleInFrame == 0) - firstSampleInFrame = pFlac->currentFrame.header.frameNumber * pFlac->maxBlockSize*channelCount; - - lastSampleInFrame = firstSampleInFrame + (pFlac->currentFrame.header.blockSize*channelCount); - if (lastSampleInFrame > 0) - lastSampleInFrame -= 1; /* Needs to be zero based. */ - - if (pFirstSampleInFrameOut) - *pFirstSampleInFrameOut = firstSampleInFrame; - if (pLastSampleInFrameOut) - *pLastSampleInFrameOut = lastSampleInFrame; -} - -static drflac_bool32 drflac__seek_to_first_frame(drflac* pFlac) -{ - drflac_bool32 result; - - drflac_assert(pFlac != NULL); - - result = drflac__seek_to_byte(&pFlac->bs, pFlac->firstFramePos); - - drflac_zero_memory(&pFlac->currentFrame, sizeof(pFlac->currentFrame)); - return result; -} - -static DRFLAC_INLINE drflac_result drflac__seek_to_next_frame(drflac* pFlac) -{ - /* This function should only ever be called while the decoder is sitting on the first byte past the FRAME_HEADER section. */ - drflac_assert(pFlac != NULL); - return drflac__seek_frame(pFlac); -} - -static drflac_bool32 drflac__seek_to_sample__brute_force(drflac* pFlac, drflac_uint64 sampleIndex) -{ - drflac_uint64 runningSampleCount = 0; - - /* We need to find the frame that contains the sample. To do this, we iterate over each frame and inspect it's header. If based on the - * header we can determine that the frame contains the sample, we do a full decode of that frame. */ - if (!drflac__seek_to_first_frame(pFlac)) - return DRFLAC_FALSE; - - for (;;) - { - drflac_uint64 sampleCountInThisFrame; - drflac_uint64 firstSampleInFrame = 0; - drflac_uint64 lastSampleInFrame = 0; - - if (!drflac__read_next_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFrame.header)) - return DRFLAC_FALSE; - - drflac__get_current_frame_sample_range(pFlac, &firstSampleInFrame, &lastSampleInFrame); - - sampleCountInThisFrame = (lastSampleInFrame - firstSampleInFrame) + 1; - if (sampleIndex < (runningSampleCount + sampleCountInThisFrame)) - { - /* The sample should be in this frame. We need to fully decode it, however if it's an invalid frame (a CRC mismatch), we need to pretend - * it never existed and keep iterating. */ - drflac_result result = drflac__decode_frame(pFlac); - if (result == DRFLAC_SUCCESS) - { - /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ - drflac_uint64 samplesToDecode = (size_t)(sampleIndex - runningSampleCount); /* <-- Safe cast because the maximum number of samples in a frame is 65535. */ - if (samplesToDecode == 0) - return DRFLAC_TRUE; - return drflac_read_s32(pFlac, samplesToDecode, NULL) != 0; /* <-- If this fails, something bad has happened (it should never fail). */ - } - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - else - { - /* It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this - * frame never existed and leave the running sample count untouched. */ - drflac_result result = drflac__seek_to_next_frame(pFlac); - if (result == DRFLAC_SUCCESS) - runningSampleCount += sampleCountInThisFrame; - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - } -} - - -static drflac_bool32 drflac__seek_to_sample__seek_table(drflac* pFlac, drflac_uint64 sampleIndex) -{ - drflac_seekpoint closestSeekpoint = {0, 0, 0}; - drflac_uint32 seekpointCount; - drflac_uint32 seekpointsRemaining; - drflac_uint64 runningSampleCount; - - drflac_assert(pFlac != NULL); - - if (pFlac->seektablePos == 0) - return DRFLAC_FALSE; - - if (!drflac__seek_to_byte(&pFlac->bs, pFlac->seektablePos)) - return DRFLAC_FALSE; - - /* The number of seek points is derived from the size of the SEEKTABLE block. */ - seekpointCount = pFlac->seektableSize / 18; /* 18 = the size of each seek point. */ - if (seekpointCount == 0) - return DRFLAC_FALSE; /* Would this ever happen? */ - - seekpointsRemaining = seekpointCount; - while (seekpointsRemaining > 0) - { - drflac_seekpoint seekpoint; - if (!drflac__read_uint64(&pFlac->bs, 64, &seekpoint.firstSample)) - break; - if (!drflac__read_uint64(&pFlac->bs, 64, &seekpoint.frameOffset)) - break; - if (!drflac__read_uint16(&pFlac->bs, 16, &seekpoint.sampleCount)) - break; - - /* Note that the seekpoint sample is based on a single channel. The input sample (sampleIndex) is based on interleaving, thus - * we need to multiple the seekpoint's sample by the channel count. */ - if (seekpoint.firstSample*pFlac->channels > sampleIndex) - break; - - closestSeekpoint = seekpoint; - seekpointsRemaining -= 1; - } - - /* At this point we should have found the seekpoint closest to our sample. We need to seek to it using basically the same - * technique as we use with the brute force method. */ - if (!drflac__seek_to_byte(&pFlac->bs, pFlac->firstFramePos + closestSeekpoint.frameOffset)) - return DRFLAC_FALSE; - - runningSampleCount = closestSeekpoint.firstSample*pFlac->channels; - for (;;) - { - drflac_uint64 sampleCountInThisFrame; - drflac_uint64 firstSampleInFrame = 0; - drflac_uint64 lastSampleInFrame = 0; - - if (!drflac__read_next_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFrame.header)) - return DRFLAC_FALSE; - - drflac__get_current_frame_sample_range(pFlac, &firstSampleInFrame, &lastSampleInFrame); - - sampleCountInThisFrame = (lastSampleInFrame - firstSampleInFrame) + 1; - if (sampleIndex < (runningSampleCount + sampleCountInThisFrame)) - { - /* The sample should be in this frame. We need to fully decode it, however if it's an invalid frame (a CRC mismatch), we need to pretend - * it never existed and keep iterating. */ - drflac_result result = drflac__decode_frame(pFlac); - - if (result == DRFLAC_SUCCESS) - { - /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ - drflac_uint64 samplesToDecode = (size_t)(sampleIndex - runningSampleCount); /* <-- Safe cast because the maximum number of samples in a frame is 65535. */ - if (samplesToDecode == 0) - return DRFLAC_TRUE; - return drflac_read_s32(pFlac, samplesToDecode, NULL) != 0; /* <-- If this fails, something bad has happened (it should never fail). */ - } - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - else - { - /* It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this - * frame never existed and leave the running sample count untouched. */ - drflac_result result = drflac__seek_to_next_frame(pFlac); - if (result == DRFLAC_SUCCESS) - runningSampleCount += sampleCountInThisFrame; - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - } -} - - -#ifndef DR_FLAC_NO_OGG -typedef struct -{ - drflac_uint8 capturePattern[4]; /* Should be "OggS" */ - drflac_uint8 structureVersion; /* Always 0. */ - drflac_uint8 headerType; - drflac_uint64 granulePosition; - drflac_uint32 serialNumber; - drflac_uint32 sequenceNumber; - drflac_uint32 checksum; - drflac_uint8 segmentCount; - drflac_uint8 segmentTable[255]; -} drflac_ogg_page_header; -#endif - -typedef struct -{ - drflac_read_proc onRead; - drflac_seek_proc onSeek; - drflac_meta_proc onMeta; - drflac_container container; - void* pUserData; - void* pUserDataMD; - drflac_uint32 sampleRate; - drflac_uint8 channels; - drflac_uint8 bitsPerSample; - drflac_uint64 totalSampleCount; - drflac_uint16 maxBlockSize; - drflac_uint64 runningFilePos; - drflac_bool32 hasStreamInfoBlock; - drflac_bool32 hasMetadataBlocks; - drflac_bs bs; /* <-- A bit streamer is required for loading data during initialization. */ - drflac_frame_header firstFrameHeader; /* <-- The header of the first frame that was read during relaxed initalization. Only set if there is no STREAMINFO block. */ - -#ifndef DR_FLAC_NO_OGG - drflac_uint32 oggSerial; - drflac_uint64 oggFirstBytePos; - drflac_ogg_page_header oggBosHeader; -#endif -} drflac_init_info; - -static DRFLAC_INLINE void drflac__decode_block_header(drflac_uint32 blockHeader, drflac_uint8* isLastBlock, drflac_uint8* blockType, drflac_uint32* blockSize) -{ - blockHeader = drflac__be2host_32(blockHeader); - *isLastBlock = (blockHeader & (0x01 << 31)) >> 31; - *blockType = (blockHeader & (0x7F << 24)) >> 24; - *blockSize = (blockHeader & 0xFFFFFF); -} - -static DRFLAC_INLINE drflac_bool32 drflac__read_and_decode_block_header(drflac_read_proc onRead, void* pUserData, drflac_uint8* isLastBlock, drflac_uint8* blockType, drflac_uint32* blockSize) -{ - drflac_uint32 blockHeader; - if (onRead(pUserData, &blockHeader, 4) != 4) { - return DRFLAC_FALSE; - } - - drflac__decode_block_header(blockHeader, isLastBlock, blockType, blockSize); - return DRFLAC_TRUE; -} - -drflac_bool32 drflac__read_streaminfo(drflac_read_proc onRead, void* pUserData, drflac_streaminfo* pStreamInfo) -{ - drflac_uint32 blockSizes; - drflac_uint64 frameSizes = 0; - drflac_uint64 importantProps; - drflac_uint8 md5[16]; - - /* min/max block size. */ - if (onRead(pUserData, &blockSizes, 4) != 4) - return DRFLAC_FALSE; - - /* min/max frame size. */ - if (onRead(pUserData, &frameSizes, 6) != 6) - return DRFLAC_FALSE; - - /* Sample rate, channels, bits per sample and total sample count. */ - if (onRead(pUserData, &importantProps, 8) != 8) - return DRFLAC_FALSE; - - /* MD5 */ - if (onRead(pUserData, md5, sizeof(md5)) != sizeof(md5)) - return DRFLAC_FALSE; - - blockSizes = drflac__be2host_32(blockSizes); - frameSizes = drflac__be2host_64(frameSizes); - importantProps = drflac__be2host_64(importantProps); - - pStreamInfo->minBlockSize = (blockSizes & 0xFFFF0000) >> 16; - pStreamInfo->maxBlockSize = blockSizes & 0x0000FFFF; - pStreamInfo->minFrameSize = (drflac_uint32)((frameSizes & (drflac_uint64)0xFFFFFF0000000000) >> 40); - pStreamInfo->maxFrameSize = (drflac_uint32)((frameSizes & (drflac_uint64)0x000000FFFFFF0000) >> 16); - pStreamInfo->sampleRate = (drflac_uint32)((importantProps & (drflac_uint64)0xFFFFF00000000000) >> 44); - pStreamInfo->channels = (drflac_uint8 )((importantProps & (drflac_uint64)0x00000E0000000000) >> 41) + 1; - pStreamInfo->bitsPerSample = (drflac_uint8 )((importantProps & (drflac_uint64)0x000001F000000000) >> 36) + 1; - pStreamInfo->totalSampleCount = (importantProps & (drflac_uint64)0x0000000FFFFFFFFF) * pStreamInfo->channels; - drflac_copy_memory(pStreamInfo->md5, md5, sizeof(md5)); - - return DRFLAC_TRUE; -} - -drflac_bool32 drflac__read_and_decode_metadata(drflac* pFlac) -{ - /* We want to keep track of the byte position in the stream of the seektable. At the time of calling this function we know that - * we'll be sitting on byte 42. */ - drflac_uint64 runningFilePos = 42; - drflac_uint64 seektablePos = 0; - drflac_uint32 seektableSize = 0; - - drflac_assert(pFlac != NULL); - - for (;;) - { - drflac_metadata metadata; - drflac_uint8 isLastBlock = 0; - drflac_uint8 blockType = 0; - drflac_uint32 blockSize = 0; - - if (!drflac__read_and_decode_block_header(pFlac->bs.onRead, pFlac->bs.pUserData, &isLastBlock, &blockType, &blockSize)) - return DRFLAC_FALSE; - runningFilePos += 4; - - - metadata.type = blockType; - metadata.pRawData = NULL; - metadata.rawDataSize = 0; - - switch (blockType) - { - case DRFLAC_METADATA_BLOCK_TYPE_APPLICATION: - { - if (pFlac->onMeta) { - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - metadata.data.application.id = drflac__be2host_32(*(drflac_uint32*)pRawData); - metadata.data.application.pData = (const void*)((drflac_uint8*)pRawData + sizeof(drflac_uint32)); - metadata.data.application.dataSize = blockSize - sizeof(drflac_uint32); - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - - case DRFLAC_METADATA_BLOCK_TYPE_SEEKTABLE: - { - seektablePos = runningFilePos; - seektableSize = blockSize; - - if (pFlac->onMeta) - { - drflac_uint32 iSeekpoint; - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - metadata.data.seektable.seekpointCount = blockSize/sizeof(drflac_seekpoint); - metadata.data.seektable.pSeekpoints = (const drflac_seekpoint*)pRawData; - - /* Endian swap. */ - for (iSeekpoint = 0; iSeekpoint < metadata.data.seektable.seekpointCount; ++iSeekpoint) { - drflac_seekpoint* pSeekpoint = (drflac_seekpoint*)pRawData + iSeekpoint; - pSeekpoint->firstSample = drflac__be2host_64(pSeekpoint->firstSample); - pSeekpoint->frameOffset = drflac__be2host_64(pSeekpoint->frameOffset); - pSeekpoint->sampleCount = drflac__be2host_16(pSeekpoint->sampleCount); - } - - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - - case DRFLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT: - { - if (pFlac->onMeta) - { - const char* pRunningData; - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - - pRunningData = (const char*)pRawData; - metadata.data.vorbis_comment.vendorLength = drflac__le2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; - metadata.data.vorbis_comment.commentCount = drflac__le2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.vorbis_comment.comments = pRunningData; - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - - case DRFLAC_METADATA_BLOCK_TYPE_CUESHEET: - { - if (pFlac->onMeta) - { - const char* pRunningData; - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - - pRunningData = (const char*)pRawData; - drflac_copy_memory(metadata.data.cuesheet.catalog, pRunningData, 128); pRunningData += 128; - metadata.data.cuesheet.leadInSampleCount = drflac__be2host_64(*(drflac_uint64*)pRunningData); pRunningData += 4; - metadata.data.cuesheet.isCD = ((pRunningData[0] & 0x80) >> 7) != 0; pRunningData += 259; - metadata.data.cuesheet.trackCount = pRunningData[0]; pRunningData += 1; - metadata.data.cuesheet.pTrackData = (const drflac_uint8*)pRunningData; - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - - case DRFLAC_METADATA_BLOCK_TYPE_PICTURE: - { - if (pFlac->onMeta) - { - const char* pRunningData; - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - - pRunningData = (const char*)pRawData; - metadata.data.picture.type = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.mimeLength = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.mime = pRunningData; pRunningData += metadata.data.picture.mimeLength; - metadata.data.picture.descriptionLength = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.description = pRunningData; - metadata.data.picture.width = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.height = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.colorDepth = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.indexColorCount = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.pictureDataSize = drflac__be2host_32(*(drflac_uint32*)pRunningData); pRunningData += 4; - metadata.data.picture.pPictureData = (const drflac_uint8*)pRunningData; - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - - case DRFLAC_METADATA_BLOCK_TYPE_PADDING: - { - if (pFlac->onMeta) - { - metadata.data.padding.unused = 0; - - /* Padding doesn't have anything meaningful in it, so just skip over it, but make sure the caller is aware of it by firing the callback. */ - if (!pFlac->bs.onSeek(pFlac->bs.pUserData, blockSize, drflac_seek_origin_current)) - isLastBlock = DRFLAC_TRUE; /* An error occured while seeking. Attempt to recover by treating this as the last block which will in turn terminate the loop. */ - else - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - } - } - break; - - case DRFLAC_METADATA_BLOCK_TYPE_INVALID: - { - /* Invalid chunk. Just skip over this one. */ - if (pFlac->onMeta) - { - if (!pFlac->bs.onSeek(pFlac->bs.pUserData, blockSize, drflac_seek_origin_current)) - isLastBlock = DRFLAC_TRUE; /* An error occured while seeking. Attempt to recover by treating this as the last block which will in turn terminate the loop. */ - } - } - - default: - { - /* It's an unknown chunk, but not necessarily invalid. There's a chance more metadata blocks might be defined later on, so we - * can at the very least report the chunk to the application and let it look at the raw data. */ - if (pFlac->onMeta) - { - void* pRawData = DRFLAC_MALLOC(blockSize); - if (pRawData == NULL) - return DRFLAC_FALSE; - - if (pFlac->bs.onRead(pFlac->bs.pUserData, pRawData, blockSize) != blockSize) { - DRFLAC_FREE(pRawData); - return DRFLAC_FALSE; - } - - metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; - pFlac->onMeta(pFlac->pUserDataMD, &metadata); - - DRFLAC_FREE(pRawData); - } - } break; - } - - /* If we're not handling metadata, just skip over the block. If we are, it will have been handled earlier in the switch statement above. */ - if (pFlac->onMeta == NULL && blockSize > 0) - { - if (!pFlac->bs.onSeek(pFlac->bs.pUserData, blockSize, drflac_seek_origin_current)) - isLastBlock = DRFLAC_TRUE; - } - - runningFilePos += blockSize; - if (isLastBlock) - break; - } - - pFlac->seektablePos = seektablePos; - pFlac->seektableSize = seektableSize; - pFlac->firstFramePos = runningFilePos; - - return DRFLAC_TRUE; -} - -drflac_bool32 drflac__init_private__native(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, void* pUserDataMD, drflac_bool32 relaxed) -{ - drflac_uint8 isLastBlock = 0; - drflac_uint8 blockType = 0; - drflac_uint32 blockSize = 0; - - (void)onSeek; - - /* Pre: The bit stream should be sitting just past the 4-byte id header. */ - - pInit->container = drflac_container_native; - - /* The first metadata block should be the STREAMINFO block. */ - if (!drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) - return DRFLAC_FALSE; - - if (blockType != DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) - { - /* We're opening in strict mode and the first block is not the STREAMINFO block. Error. */ - if (!relaxed) - return DRFLAC_FALSE; - - /* Relaxed mode. To open from here we need to just find the first frame and set the sample rate, etc. to whatever is defined - * for that frame. */ - pInit->hasStreamInfoBlock = DRFLAC_FALSE; - pInit->hasMetadataBlocks = DRFLAC_FALSE; - - if (!drflac__read_next_frame_header(&pInit->bs, 0, &pInit->firstFrameHeader)) - return DRFLAC_FALSE; /* Couldn't find a frame. */ - - if (pInit->firstFrameHeader.bitsPerSample == 0) - return DRFLAC_FALSE; /* Failed to initialize because the first frame depends on the STREAMINFO block, which does not exist. */ - - pInit->sampleRate = pInit->firstFrameHeader.sampleRate; - pInit->channels = drflac__get_channel_count_from_channel_assignment(pInit->firstFrameHeader.channelAssignment); - pInit->bitsPerSample = pInit->firstFrameHeader.bitsPerSample; - pInit->maxBlockSize = 65535; /* <-- See notes here: https://xiph.org/flac/format.html#metadata_block_streaminfo */ - return DRFLAC_TRUE; - } - else - { - drflac_streaminfo streaminfo; - if (!drflac__read_streaminfo(onRead, pUserData, &streaminfo)) - return DRFLAC_FALSE; - - pInit->hasStreamInfoBlock = DRFLAC_TRUE; - pInit->sampleRate = streaminfo.sampleRate; - pInit->channels = streaminfo.channels; - pInit->bitsPerSample = streaminfo.bitsPerSample; - pInit->totalSampleCount = streaminfo.totalSampleCount; - pInit->maxBlockSize = streaminfo.maxBlockSize; /* Don't care about the min block size - only the max (used for determining the size of the memory allocation). */ - pInit->hasMetadataBlocks = !isLastBlock; - - if (onMeta) { - drflac_metadata metadata; - metadata.type = DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO; - metadata.pRawData = NULL; - metadata.rawDataSize = 0; - metadata.data.streaminfo = streaminfo; - onMeta(pUserDataMD, &metadata); - } - - return DRFLAC_TRUE; - } -} - -#ifndef DR_FLAC_NO_OGG -#define DRFLAC_OGG_MAX_PAGE_SIZE 65307 -#define DRFLAC_OGG_CAPTURE_PATTERN_CRC32 1605413199 /* CRC-32 of "OggS". */ - -typedef enum -{ - drflac_ogg_recover_on_crc_mismatch, - drflac_ogg_fail_on_crc_mismatch -} drflac_ogg_crc_mismatch_recovery; - - -static drflac_uint32 drflac__crc32_table[] = { - 0x00000000L, 0x04C11DB7L, 0x09823B6EL, 0x0D4326D9L, - 0x130476DCL, 0x17C56B6BL, 0x1A864DB2L, 0x1E475005L, - 0x2608EDB8L, 0x22C9F00FL, 0x2F8AD6D6L, 0x2B4BCB61L, - 0x350C9B64L, 0x31CD86D3L, 0x3C8EA00AL, 0x384FBDBDL, - 0x4C11DB70L, 0x48D0C6C7L, 0x4593E01EL, 0x4152FDA9L, - 0x5F15ADACL, 0x5BD4B01BL, 0x569796C2L, 0x52568B75L, - 0x6A1936C8L, 0x6ED82B7FL, 0x639B0DA6L, 0x675A1011L, - 0x791D4014L, 0x7DDC5DA3L, 0x709F7B7AL, 0x745E66CDL, - 0x9823B6E0L, 0x9CE2AB57L, 0x91A18D8EL, 0x95609039L, - 0x8B27C03CL, 0x8FE6DD8BL, 0x82A5FB52L, 0x8664E6E5L, - 0xBE2B5B58L, 0xBAEA46EFL, 0xB7A96036L, 0xB3687D81L, - 0xAD2F2D84L, 0xA9EE3033L, 0xA4AD16EAL, 0xA06C0B5DL, - 0xD4326D90L, 0xD0F37027L, 0xDDB056FEL, 0xD9714B49L, - 0xC7361B4CL, 0xC3F706FBL, 0xCEB42022L, 0xCA753D95L, - 0xF23A8028L, 0xF6FB9D9FL, 0xFBB8BB46L, 0xFF79A6F1L, - 0xE13EF6F4L, 0xE5FFEB43L, 0xE8BCCD9AL, 0xEC7DD02DL, - 0x34867077L, 0x30476DC0L, 0x3D044B19L, 0x39C556AEL, - 0x278206ABL, 0x23431B1CL, 0x2E003DC5L, 0x2AC12072L, - 0x128E9DCFL, 0x164F8078L, 0x1B0CA6A1L, 0x1FCDBB16L, - 0x018AEB13L, 0x054BF6A4L, 0x0808D07DL, 0x0CC9CDCAL, - 0x7897AB07L, 0x7C56B6B0L, 0x71159069L, 0x75D48DDEL, - 0x6B93DDDBL, 0x6F52C06CL, 0x6211E6B5L, 0x66D0FB02L, - 0x5E9F46BFL, 0x5A5E5B08L, 0x571D7DD1L, 0x53DC6066L, - 0x4D9B3063L, 0x495A2DD4L, 0x44190B0DL, 0x40D816BAL, - 0xACA5C697L, 0xA864DB20L, 0xA527FDF9L, 0xA1E6E04EL, - 0xBFA1B04BL, 0xBB60ADFCL, 0xB6238B25L, 0xB2E29692L, - 0x8AAD2B2FL, 0x8E6C3698L, 0x832F1041L, 0x87EE0DF6L, - 0x99A95DF3L, 0x9D684044L, 0x902B669DL, 0x94EA7B2AL, - 0xE0B41DE7L, 0xE4750050L, 0xE9362689L, 0xEDF73B3EL, - 0xF3B06B3BL, 0xF771768CL, 0xFA325055L, 0xFEF34DE2L, - 0xC6BCF05FL, 0xC27DEDE8L, 0xCF3ECB31L, 0xCBFFD686L, - 0xD5B88683L, 0xD1799B34L, 0xDC3ABDEDL, 0xD8FBA05AL, - 0x690CE0EEL, 0x6DCDFD59L, 0x608EDB80L, 0x644FC637L, - 0x7A089632L, 0x7EC98B85L, 0x738AAD5CL, 0x774BB0EBL, - 0x4F040D56L, 0x4BC510E1L, 0x46863638L, 0x42472B8FL, - 0x5C007B8AL, 0x58C1663DL, 0x558240E4L, 0x51435D53L, - 0x251D3B9EL, 0x21DC2629L, 0x2C9F00F0L, 0x285E1D47L, - 0x36194D42L, 0x32D850F5L, 0x3F9B762CL, 0x3B5A6B9BL, - 0x0315D626L, 0x07D4CB91L, 0x0A97ED48L, 0x0E56F0FFL, - 0x1011A0FAL, 0x14D0BD4DL, 0x19939B94L, 0x1D528623L, - 0xF12F560EL, 0xF5EE4BB9L, 0xF8AD6D60L, 0xFC6C70D7L, - 0xE22B20D2L, 0xE6EA3D65L, 0xEBA91BBCL, 0xEF68060BL, - 0xD727BBB6L, 0xD3E6A601L, 0xDEA580D8L, 0xDA649D6FL, - 0xC423CD6AL, 0xC0E2D0DDL, 0xCDA1F604L, 0xC960EBB3L, - 0xBD3E8D7EL, 0xB9FF90C9L, 0xB4BCB610L, 0xB07DABA7L, - 0xAE3AFBA2L, 0xAAFBE615L, 0xA7B8C0CCL, 0xA379DD7BL, - 0x9B3660C6L, 0x9FF77D71L, 0x92B45BA8L, 0x9675461FL, - 0x8832161AL, 0x8CF30BADL, 0x81B02D74L, 0x857130C3L, - 0x5D8A9099L, 0x594B8D2EL, 0x5408ABF7L, 0x50C9B640L, - 0x4E8EE645L, 0x4A4FFBF2L, 0x470CDD2BL, 0x43CDC09CL, - 0x7B827D21L, 0x7F436096L, 0x7200464FL, 0x76C15BF8L, - 0x68860BFDL, 0x6C47164AL, 0x61043093L, 0x65C52D24L, - 0x119B4BE9L, 0x155A565EL, 0x18197087L, 0x1CD86D30L, - 0x029F3D35L, 0x065E2082L, 0x0B1D065BL, 0x0FDC1BECL, - 0x3793A651L, 0x3352BBE6L, 0x3E119D3FL, 0x3AD08088L, - 0x2497D08DL, 0x2056CD3AL, 0x2D15EBE3L, 0x29D4F654L, - 0xC5A92679L, 0xC1683BCEL, 0xCC2B1D17L, 0xC8EA00A0L, - 0xD6AD50A5L, 0xD26C4D12L, 0xDF2F6BCBL, 0xDBEE767CL, - 0xE3A1CBC1L, 0xE760D676L, 0xEA23F0AFL, 0xEEE2ED18L, - 0xF0A5BD1DL, 0xF464A0AAL, 0xF9278673L, 0xFDE69BC4L, - 0x89B8FD09L, 0x8D79E0BEL, 0x803AC667L, 0x84FBDBD0L, - 0x9ABC8BD5L, 0x9E7D9662L, 0x933EB0BBL, 0x97FFAD0CL, - 0xAFB010B1L, 0xAB710D06L, 0xA6322BDFL, 0xA2F33668L, - 0xBCB4666DL, 0xB8757BDAL, 0xB5365D03L, 0xB1F740B4L -}; - -static DRFLAC_INLINE drflac_uint32 drflac_crc32_byte(drflac_uint32 crc32, drflac_uint8 data) -{ -#ifndef DR_FLAC_NO_CRC - return (crc32 << 8) ^ drflac__crc32_table[(drflac_uint8)((crc32 >> 24) & 0xFF) ^ data]; -#else - (void)data; - return crc32; -#endif -} - -#if 0 -static DRFLAC_INLINE drflac_uint32 drflac_crc32_uint32(drflac_uint32 crc32, drflac_uint32 data) -{ - crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 24) & 0xFF)); - crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 16) & 0xFF)); - crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 8) & 0xFF)); - crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 0) & 0xFF)); - return crc32; -} - -static DRFLAC_INLINE drflac_uint32 drflac_crc32_uint64(drflac_uint32 crc32, drflac_uint64 data) -{ - crc32 = drflac_crc32_uint32(crc32, (drflac_uint32)((data >> 32) & 0xFFFFFFFF)); - crc32 = drflac_crc32_uint32(crc32, (drflac_uint32)((data >> 0) & 0xFFFFFFFF)); - return crc32; -} -#endif - -static DRFLAC_INLINE drflac_uint32 drflac_crc32_buffer(drflac_uint32 crc32, drflac_uint8* pData, drflac_uint32 dataSize) -{ - drflac_uint32 i; - - /* This can be optimized. */ - for (i = 0; i < dataSize; ++i) - crc32 = drflac_crc32_byte(crc32, pData[i]); - return crc32; -} - - -static DRFLAC_INLINE drflac_bool32 drflac_ogg__is_capture_pattern(drflac_uint8 pattern[4]) -{ - return pattern[0] == 'O' && pattern[1] == 'g' && pattern[2] == 'g' && pattern[3] == 'S'; -} - -static DRFLAC_INLINE drflac_uint32 drflac_ogg__get_page_header_size(drflac_ogg_page_header* pHeader) -{ - return 27 + pHeader->segmentCount; -} - -static DRFLAC_INLINE drflac_uint32 drflac_ogg__get_page_body_size(drflac_ogg_page_header* pHeader) -{ - int i; - drflac_uint32 pageBodySize = 0; - - for (i = 0; i < pHeader->segmentCount; ++i) - pageBodySize += pHeader->segmentTable[i]; - - return pageBodySize; -} - -drflac_result drflac_ogg__read_page_header_after_capture_pattern(drflac_read_proc onRead, void* pUserData, drflac_ogg_page_header* pHeader, drflac_uint32* pBytesRead, drflac_uint32* pCRC32) -{ - drflac_uint32 i; - drflac_uint8 data[23]; - - drflac_assert(*pCRC32 == DRFLAC_OGG_CAPTURE_PATTERN_CRC32); - - if (onRead(pUserData, data, 23) != 23) - return DRFLAC_END_OF_STREAM; - *pBytesRead += 23; - - pHeader->structureVersion = data[0]; - pHeader->headerType = data[1]; - drflac_copy_memory(&pHeader->granulePosition, &data[ 2], 8); - drflac_copy_memory(&pHeader->serialNumber, &data[10], 4); - drflac_copy_memory(&pHeader->sequenceNumber, &data[14], 4); - drflac_copy_memory(&pHeader->checksum, &data[18], 4); - pHeader->segmentCount = data[22]; - - /* Calculate the CRC. Note that for the calculation the checksum part of the page needs to be set to 0. */ - data[18] = 0; - data[19] = 0; - data[20] = 0; - data[21] = 0; - - for (i = 0; i < 23; ++i) - *pCRC32 = drflac_crc32_byte(*pCRC32, data[i]); - - if (onRead(pUserData, pHeader->segmentTable, pHeader->segmentCount) != pHeader->segmentCount) - return DRFLAC_END_OF_STREAM; - - *pBytesRead += pHeader->segmentCount; - - for (i = 0; i < pHeader->segmentCount; ++i) - *pCRC32 = drflac_crc32_byte(*pCRC32, pHeader->segmentTable[i]); - - return DRFLAC_SUCCESS; -} - -drflac_result drflac_ogg__read_page_header(drflac_read_proc onRead, void* pUserData, drflac_ogg_page_header* pHeader, drflac_uint32* pBytesRead, drflac_uint32* pCRC32) -{ - drflac_uint8 id[4]; - - *pBytesRead = 0; - - if (onRead(pUserData, id, 4) != 4) - return DRFLAC_END_OF_STREAM; - *pBytesRead += 4; - - /* We need to read byte-by-byte until we find the OggS capture pattern. */ - for (;;) - { - if (drflac_ogg__is_capture_pattern(id)) - { - drflac_result result; - *pCRC32 = DRFLAC_OGG_CAPTURE_PATTERN_CRC32; - - result = drflac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, pHeader, pBytesRead, pCRC32); - if (result == DRFLAC_SUCCESS) - return DRFLAC_SUCCESS; - - if (result == DRFLAC_CRC_MISMATCH) - continue; - return result; - } - else - { - /* The first 4 bytes did not equal the capture pattern. Read the next byte and try again. */ - id[0] = id[1]; - id[1] = id[2]; - id[2] = id[3]; - if (onRead(pUserData, &id[3], 1) != 1) - return DRFLAC_END_OF_STREAM; - *pBytesRead += 1; - } - } -} - - -/* The main part of the Ogg encapsulation is the conversion from the physical Ogg bitstream to the native FLAC bitstream. It works - * in three general stages: Ogg Physical Bitstream -> Ogg/FLAC Logical Bitstream -> FLAC Native Bitstream. dr_flac is architecured - * in such a way that the core sections assume everything is delivered in native format. Therefore, for each encapsulation type - * dr_flac is supporting there needs to be a layer sitting on top of the onRead and onSeek callbacks that ensures the bits read from - * the physical Ogg bitstream are converted and delivered in native FLAC format. */ -typedef struct -{ - drflac_read_proc onRead; /* The original onRead callback from drflac_open() and family. */ - drflac_seek_proc onSeek; /* The original onSeek callback from drflac_open() and family. */ - void* pUserData; /* The user data passed on onRead and onSeek. This is the user data that was passed on drflac_open() and family. */ - drflac_uint64 currentBytePos; /* The position of the byte we are sitting on in the physical byte stream. Used for efficient seeking. */ - drflac_uint64 firstBytePos; /* The position of the first byte in the physical bitstream. Points to the start of the "OggS" identifier of the FLAC bos page. */ - drflac_uint32 serialNumber; /* The serial number of the FLAC audio pages. This is determined by the initial header page that was read during initialization. */ - drflac_ogg_page_header bosPageHeader; /* Used for seeking. */ - drflac_ogg_page_header currentPageHeader; - drflac_uint32 bytesRemainingInPage; - drflac_uint32 pageDataSize; - drflac_uint8 pageData[DRFLAC_OGG_MAX_PAGE_SIZE]; -} drflac_oggbs; /* oggbs = Ogg Bitstream */ - -static size_t drflac_oggbs__read_physical(drflac_oggbs* oggbs, void* bufferOut, size_t bytesToRead) -{ - size_t bytesActuallyRead = oggbs->onRead(oggbs->pUserData, bufferOut, bytesToRead); - oggbs->currentBytePos += bytesActuallyRead; - - return bytesActuallyRead; -} - -static drflac_bool32 drflac_oggbs__seek_physical(drflac_oggbs* oggbs, drflac_uint64 offset, drflac_seek_origin origin) -{ - if (origin == drflac_seek_origin_start) - { - if (offset <= 0x7FFFFFFF) - { - if (!oggbs->onSeek(oggbs->pUserData, (int)offset, drflac_seek_origin_start)) - return DRFLAC_FALSE; - oggbs->currentBytePos = offset; - - return DRFLAC_TRUE; - } - else - { - if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, drflac_seek_origin_start)) - return DRFLAC_FALSE; - oggbs->currentBytePos = offset; - - return drflac_oggbs__seek_physical(oggbs, offset - 0x7FFFFFFF, drflac_seek_origin_current); - } - } - else - { - while (offset > 0x7FFFFFFF) - { - if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, drflac_seek_origin_current)) - return DRFLAC_FALSE; - oggbs->currentBytePos += 0x7FFFFFFF; - offset -= 0x7FFFFFFF; - } - - if (!oggbs->onSeek(oggbs->pUserData, (int)offset, drflac_seek_origin_current)) /* <-- Safe cast thanks to the loop above. */ - return DRFLAC_FALSE; - oggbs->currentBytePos += offset; - - return DRFLAC_TRUE; - } -} - -static drflac_bool32 drflac_oggbs__goto_next_page(drflac_oggbs* oggbs, drflac_ogg_crc_mismatch_recovery recoveryMethod) -{ - drflac_ogg_page_header header; - for (;;) { -#ifndef DR_FLAC_NO_CRC - drflac_uint32 actualCRC32; -#endif - drflac_uint32 pageBodySize; - drflac_uint32 crc32 = 0; - drflac_uint32 bytesRead; - if (drflac_ogg__read_page_header(oggbs->onRead, oggbs->pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) - return DRFLAC_FALSE; - oggbs->currentBytePos += bytesRead; - - pageBodySize = drflac_ogg__get_page_body_size(&header); - if (pageBodySize > DRFLAC_OGG_MAX_PAGE_SIZE) - continue; /* Invalid page size. Assume it's corrupted and just move to the next page. */ - - if (header.serialNumber != oggbs->serialNumber) - { - /* It's not a FLAC page. Skip it. */ - if (pageBodySize > 0 && !drflac_oggbs__seek_physical(oggbs, pageBodySize, drflac_seek_origin_current)) - return DRFLAC_FALSE; - continue; - } - - /* We need to read the entire page and then do a CRC check on it. If there's a CRC mismatch we need to skip this page. */ - if (drflac_oggbs__read_physical(oggbs, oggbs->pageData, pageBodySize) != pageBodySize) - return DRFLAC_FALSE; - oggbs->pageDataSize = pageBodySize; - -#ifndef DR_FLAC_NO_CRC - actualCRC32 = drflac_crc32_buffer(crc32, oggbs->pageData, oggbs->pageDataSize); - if (actualCRC32 != header.checksum) - { - if (recoveryMethod == drflac_ogg_recover_on_crc_mismatch) - continue; /* CRC mismatch. Skip this page. */ - - /* Even though we are failing on a CRC mismatch, we still want our stream to be in a good state. Therefore we - * go to the next valid page to ensure we're in a good state, but return false to let the caller know that the - * seek did not fully complete. */ - drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch); - return DRFLAC_FALSE; - } -#endif - - oggbs->currentPageHeader = header; - oggbs->bytesRemainingInPage = pageBodySize; - return DRFLAC_TRUE; - } -} - -/* Function below is unused at the moment, but I might be re-adding it later. */ -#if 0 -static drflac_uint8 drflac_oggbs__get_current_segment_index(drflac_oggbs* oggbs, drflac_uint8* pBytesRemainingInSeg) -{ - drflac_uint32 bytesConsumedInPage = drflac_ogg__get_page_body_size(&oggbs->currentPageHeader) - oggbs->bytesRemainingInPage; - drflac_uint8 iSeg = 0; - drflac_uint32 iByte = 0; - while (iByte < bytesConsumedInPage) - { - drflac_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg]; - if (iByte + segmentSize > bytesConsumedInPage) - break; - - iSeg += 1; - iByte += segmentSize; - } - - *pBytesRemainingInSeg = oggbs->currentPageHeader.segmentTable[iSeg] - (drflac_uint8)(bytesConsumedInPage - iByte); - return iSeg; -} - -static drflac_bool32 drflac_oggbs__seek_to_next_packet(drflac_oggbs* oggbs) -{ - /* The current packet ends when we get to the segment with a lacing value of < 255 which is not at the end of a page. */ - for (;;) { - drflac_bool32 atEndOfPage = DRFLAC_FALSE; - - drflac_uint8 bytesRemainingInSeg; - drflac_uint8 iFirstSeg = drflac_oggbs__get_current_segment_index(oggbs, &bytesRemainingInSeg); - - drflac_uint32 bytesToEndOfPacketOrPage = bytesRemainingInSeg; - for (drflac_uint8 iSeg = iFirstSeg; iSeg < oggbs->currentPageHeader.segmentCount; ++iSeg) { - drflac_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg]; - if (segmentSize < 255) - { - if (iSeg == oggbs->currentPageHeader.segmentCount-1) - atEndOfPage = DRFLAC_TRUE; - - break; - } - - bytesToEndOfPacketOrPage += segmentSize; - } - - /* At this point we will have found either the packet or the end of the page. If were at the end of the page we'll - * want to load the next page and keep searching for the end of the packet. */ - drflac_oggbs__seek_physical(oggbs, bytesToEndOfPacketOrPage, drflac_seek_origin_current); - oggbs->bytesRemainingInPage -= bytesToEndOfPacketOrPage; - - if (atEndOfPage) - { - /* We're potentially at the next packet, but we need to check the next page first to be sure because the packet may - * straddle pages. */ - if (!drflac_oggbs__goto_next_page(oggbs)) - return DRFLAC_FALSE; - - /* If it's a fresh packet it most likely means we're at the next packet. */ - if ((oggbs->currentPageHeader.headerType & 0x01) == 0) - return DRFLAC_TRUE; - } - else - { - /* We're at the next packet. */ - return DRFLAC_TRUE; - } - } -} - -static drflac_bool32 drflac_oggbs__seek_to_next_frame(drflac_oggbs* oggbs) -{ - /* The bitstream should be sitting on the first byte just after the header of the frame. - - * What we're actually doing here is seeking to the start of the next packet. */ - return drflac_oggbs__seek_to_next_packet(oggbs); -} -#endif - -static size_t drflac__on_read_ogg(void* pUserData, void* bufferOut, size_t bytesToRead) -{ - size_t bytesRead = 0; - drflac_uint8* pRunningBufferOut = NULL; - drflac_oggbs* oggbs = (drflac_oggbs*)pUserData; - - drflac_assert(oggbs != NULL); - - pRunningBufferOut = (drflac_uint8*)bufferOut; - - /* Reading is done page-by-page. If we've run out of bytes in the page we need to move to the next one. */ - while (bytesRead < bytesToRead) - { - size_t bytesRemainingToRead = bytesToRead - bytesRead; - - if (oggbs->bytesRemainingInPage >= bytesRemainingToRead) { - drflac_copy_memory(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), bytesRemainingToRead); - bytesRead += bytesRemainingToRead; - oggbs->bytesRemainingInPage -= (drflac_uint32)bytesRemainingToRead; - break; - } - - /* If we get here it means some of the requested data is contained in the next pages. */ - if (oggbs->bytesRemainingInPage > 0) - { - drflac_copy_memory(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), oggbs->bytesRemainingInPage); - bytesRead += oggbs->bytesRemainingInPage; - pRunningBufferOut += oggbs->bytesRemainingInPage; - oggbs->bytesRemainingInPage = 0; - } - - drflac_assert(bytesRemainingToRead > 0); - if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) - break; /* Failed to go to the next page. Might have simply hit the end of the stream. */ - } - - return bytesRead; -} - -static drflac_bool32 drflac__on_seek_ogg(void* pUserData, int offset, drflac_seek_origin origin) -{ - int bytesSeeked = 0; - drflac_oggbs* oggbs = (drflac_oggbs*)pUserData; - - drflac_assert(oggbs != NULL); - drflac_assert(offset > 0 || (offset == 0 && origin == drflac_seek_origin_start)); - - /* Seeking is always forward which makes things a lot simpler. */ - if (origin == drflac_seek_origin_start) - { - if (!drflac_oggbs__seek_physical(oggbs, (int)oggbs->firstBytePos, drflac_seek_origin_start)) - return DRFLAC_FALSE; - - if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_fail_on_crc_mismatch)) - return DRFLAC_FALSE; - - return drflac__on_seek_ogg(pUserData, offset, drflac_seek_origin_current); - } - - drflac_assert(origin == drflac_seek_origin_current); - - while (bytesSeeked < offset) - { - int bytesRemainingToSeek = offset - bytesSeeked; - drflac_assert(bytesRemainingToSeek >= 0); - - if (oggbs->bytesRemainingInPage >= (size_t)bytesRemainingToSeek) - { - oggbs->bytesRemainingInPage -= bytesRemainingToSeek; - break; - } - - /* If we get here it means some of the requested data is contained in the next pages. */ - if (oggbs->bytesRemainingInPage > 0) - { - bytesSeeked += (int)oggbs->bytesRemainingInPage; - oggbs->bytesRemainingInPage = 0; - } - - drflac_assert(bytesRemainingToSeek > 0); - - /* Failed to go to the next page. We either hit the end of the stream or had a CRC mismatch. */ - if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_fail_on_crc_mismatch)) - return DRFLAC_FALSE; - } - - return DRFLAC_TRUE; -} - -drflac_bool32 drflac_ogg__seek_to_sample(drflac* pFlac, drflac_uint64 sampleIndex) -{ - drflac_uint64 runningSampleCount; - drflac_uint64 runningFrameBytePos; - drflac_uint64 runningGranulePosition = 0; - drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; - - drflac_uint64 originalBytePos = oggbs->currentBytePos; /* For recovery. */ - - /* First seek to the first frame. */ - if (!drflac__seek_to_byte(&pFlac->bs, pFlac->firstFramePos)) - return DRFLAC_FALSE; - oggbs->bytesRemainingInPage = 0; - - for (;;) - { - if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) { - drflac_oggbs__seek_physical(oggbs, originalBytePos, drflac_seek_origin_start); - return DRFLAC_FALSE; /* Never did find that sample... */ - } - - runningFrameBytePos = oggbs->currentBytePos - drflac_ogg__get_page_header_size(&oggbs->currentPageHeader) - oggbs->pageDataSize; - if (oggbs->currentPageHeader.granulePosition*pFlac->channels >= sampleIndex) - break; /* The sample is somewhere in the previous page. */ - - /* At this point we know the sample is not in the previous page. It could possibly be in this page. For simplicity we - * disregard any pages that do not begin a fresh packet. */ - if ((oggbs->currentPageHeader.headerType & 0x01) == 0) - { /* <-- Is it a fresh page? */ - if (oggbs->currentPageHeader.segmentTable[0] >= 2) - { - drflac_uint8 firstBytesInPage[2]; - firstBytesInPage[0] = oggbs->pageData[0]; - firstBytesInPage[1] = oggbs->pageData[1]; - - if ((firstBytesInPage[0] == 0xFF) && (firstBytesInPage[1] & 0xFC) == 0xF8) /* <-- Does the page begin with a frame's sync code? */ - runningGranulePosition = oggbs->currentPageHeader.granulePosition*pFlac->channels; - - continue; - } - } - } - - - /* We found the page that that is closest to the sample, so now we need to find it. The first thing to do is seek to the - * start of that page. In the loop above we checked that it was a fresh page which means this page is also the start of - * a new frame. This property means that after we've seeked to the page we can immediately start looping over frames until - * we find the one containing the target sample. */ - if (!drflac_oggbs__seek_physical(oggbs, runningFrameBytePos, drflac_seek_origin_start)) - return DRFLAC_FALSE; - if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) - return DRFLAC_FALSE; - - /* At this point we'll be sitting on the first byte of the frame header of the first frame in the page. We just keep - * looping over these frames until we find the one containing the sample we're after. */ - runningSampleCount = runningGranulePosition; - for (;;) - { - drflac_uint64 sampleCountInThisFrame; - drflac_uint64 firstSampleInFrame = 0; - drflac_uint64 lastSampleInFrame = 0; - /* There are two ways to find the sample and seek past irrelevant frames: - * 1) Use the native FLAC decoder. - * 2) Use Ogg's framing system. - * - * Both of these options have their own pros and cons. Using the native FLAC decoder is slower because it needs to - * do a full decode of the frame. Using Ogg's framing system is faster, but more complicated and involves some code - * duplication for the decoding of frame headers. - * - * Another thing to consider is that using the Ogg framing system will perform direct seeking of the physical Ogg - * bitstream. This is important to consider because it means we cannot read data from the drflac_bs object using the - * standard drflac__*() APIs because that will read in extra data for it's own internal caching which in turn breaks - * the positioning of the read pointer of the physical Ogg bitstream. Therefore, anything that would normally be read - * using the native FLAC decoding APIs, such as drflac__read_next_frame_header(), need to be re-implemented so as to - * avoid the use of the drflac_bs object. - * - * Considering these issues, I have decided to use the slower native FLAC decoding method for the following reasons: - * 1) Seeking is already partially accellerated using Ogg's paging system in the code block above. - * 2) Seeking in an Ogg encapsulated FLAC stream is probably quite uncommon. - * 3) Simplicity. - */ - if (!drflac__read_next_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFrame.header)) - return DRFLAC_FALSE; - - drflac__get_current_frame_sample_range(pFlac, &firstSampleInFrame, &lastSampleInFrame); - - sampleCountInThisFrame = (lastSampleInFrame - firstSampleInFrame) + 1; - if (sampleIndex < (runningSampleCount + sampleCountInThisFrame)) { - /* The sample should be in this frame. We need to fully decode it, however if it's an invalid frame (a CRC mismatch), we need to pretend - * it never existed and keep iterating. */ - drflac_result result = drflac__decode_frame(pFlac); - if (result == DRFLAC_SUCCESS) { - /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ - drflac_uint64 samplesToDecode = (size_t)(sampleIndex - runningSampleCount); /* <-- Safe cast because the maximum number of samples in a frame is 65535. */ - if (samplesToDecode == 0) - return DRFLAC_TRUE; - return drflac_read_s32(pFlac, samplesToDecode, NULL) != 0; /* <-- If this fails, something bad has happened (it should never fail). */ - } - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - else - { - /* It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this - * frame never existed and leave the running sample count untouched. */ - drflac_result result = drflac__seek_to_next_frame(pFlac); - if (result == DRFLAC_SUCCESS) - runningSampleCount += sampleCountInThisFrame; - else - { - if (result == DRFLAC_CRC_MISMATCH) - continue; /* CRC mismatch. Pretend this frame never existed. */ - return DRFLAC_FALSE; - } - } - } -} - - -drflac_bool32 drflac__init_private__ogg(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, void* pUserDataMD, drflac_bool32 relaxed) -{ - /* We'll get here if the first 4 bytes of the stream were the OggS capture pattern, however it doesn't necessarily mean the - * stream includes FLAC encoded audio. To check for this we need to scan the beginning-of-stream page markers and check if - * any match the FLAC specification. Important to keep in mind that the stream may be multiplexed. */ - drflac_ogg_page_header header; - - drflac_uint32 crc32 = DRFLAC_OGG_CAPTURE_PATTERN_CRC32; - drflac_uint32 bytesRead = 0; - /* Pre: The bit stream should be sitting just past the 4-byte OggS capture pattern. */ - (void)relaxed; - - pInit->container = drflac_container_ogg; - pInit->oggFirstBytePos = 0; - - if (drflac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) - return DRFLAC_FALSE; - pInit->runningFilePos += bytesRead; - - for (;;) - { - int pageBodySize; - /* Break if we're past the beginning of stream page. */ - if ((header.headerType & 0x02) == 0) - return DRFLAC_FALSE; - - /* Check if it's a FLAC header. */ - pageBodySize = drflac_ogg__get_page_body_size(&header); - if (pageBodySize == 51) - { - /* 51 = the lacing value of the FLAC header packet. - * It could be a FLAC page... */ - drflac_uint32 bytesRemainingInPage = pageBodySize; - - drflac_uint8 packetType; - if (onRead(pUserData, &packetType, 1) != 1) - return DRFLAC_FALSE; - - bytesRemainingInPage -= 1; - if (packetType == 0x7F) - { - /* Increasingly more likely to be a FLAC page... */ - drflac_uint8 sig[4]; - if (onRead(pUserData, sig, 4) != 4) - return DRFLAC_FALSE; - - bytesRemainingInPage -= 4; - if (sig[0] == 'F' && sig[1] == 'L' && sig[2] == 'A' && sig[3] == 'C') { - /* Almost certainly a FLAC page... */ - drflac_uint8 mappingVersion[2]; - if (onRead(pUserData, mappingVersion, 2) != 2) - return DRFLAC_FALSE; - - if (mappingVersion[0] != 1) - return DRFLAC_FALSE; /* Only supporting version 1.x of the Ogg mapping. */ - - /* The next 2 bytes are the non-audio packets, not including this one. We don't care about this because we're going to - * be handling it in a generic way based on the serial number and packet types. */ - if (!onSeek(pUserData, 2, drflac_seek_origin_current)) - return DRFLAC_FALSE; - - /* Expecting the native FLAC signature "fLaC". */ - if (onRead(pUserData, sig, 4) != 4) - return DRFLAC_FALSE; - - if (sig[0] == 'f' && sig[1] == 'L' && sig[2] == 'a' && sig[3] == 'C') - { - drflac_streaminfo streaminfo; - /* The remaining data in the page should be the STREAMINFO block. */ - drflac_uint8 isLastBlock = 0; - drflac_uint8 blockType = 0; - drflac_uint32 blockSize = 0; - if (!drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) - return DRFLAC_FALSE; - - if (blockType != DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) - return DRFLAC_FALSE; /* Invalid block type. First block must be the STREAMINFO block. */ - - if (drflac__read_streaminfo(onRead, pUserData, &streaminfo)) { - /* Success! */ - pInit->hasStreamInfoBlock = DRFLAC_TRUE; - pInit->sampleRate = streaminfo.sampleRate; - pInit->channels = streaminfo.channels; - pInit->bitsPerSample = streaminfo.bitsPerSample; - pInit->totalSampleCount = streaminfo.totalSampleCount; - pInit->maxBlockSize = streaminfo.maxBlockSize; - pInit->hasMetadataBlocks = !isLastBlock; - - if (onMeta) { - drflac_metadata metadata; - metadata.type = DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO; - metadata.pRawData = NULL; - metadata.rawDataSize = 0; - metadata.data.streaminfo = streaminfo; - onMeta(pUserDataMD, &metadata); - } - - pInit->runningFilePos += pageBodySize; - pInit->oggFirstBytePos = pInit->runningFilePos - 79; /* Subtracting 79 will place us right on top of the "OggS" identifier of the FLAC bos page. */ - pInit->oggSerial = header.serialNumber; - pInit->oggBosHeader = header; - break; - } - else - { - /* Failed to read STREAMINFO block. Aww, so close... */ - return DRFLAC_FALSE; - } - } - else - { - /* Invalid file. */ - return DRFLAC_FALSE; - } - } - else - { - /* Not a FLAC header. Skip it. */ - if (!onSeek(pUserData, bytesRemainingInPage, drflac_seek_origin_current)) - return DRFLAC_FALSE; - } - } - else - { - /* Not a FLAC header. Seek past the entire page and move on to the next. */ - if (!onSeek(pUserData, bytesRemainingInPage, drflac_seek_origin_current)) - return DRFLAC_FALSE; - } - } - else - { - if (!onSeek(pUserData, pageBodySize, drflac_seek_origin_current)) - return DRFLAC_FALSE; - } - - pInit->runningFilePos += pageBodySize; - - /* Read the header of the next page. */ - if (drflac_ogg__read_page_header(onRead, pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) - return DRFLAC_FALSE; - pInit->runningFilePos += bytesRead; - } - - - /* If we get here it means we found a FLAC audio stream. We should be sitting on the first byte of the header of the next page. The next - * packets in the FLAC logical stream contain the metadata. The only thing left to do in the initialiation phase for Ogg is to create the - * Ogg bistream object. */ - pInit->hasMetadataBlocks = DRFLAC_TRUE; /* <-- Always have at least VORBIS_COMMENT metadata block. */ - return DRFLAC_TRUE; -} -#endif - -drflac_bool32 drflac__init_private(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, void* pUserDataMD) -{ - drflac_uint8 id[4]; - drflac_bool32 relaxed; - - if (pInit == NULL || onRead == NULL || onSeek == NULL) - return DRFLAC_FALSE; - - drflac_zero_memory(pInit, sizeof(*pInit)); - pInit->onRead = onRead; - pInit->onSeek = onSeek; - pInit->onMeta = onMeta; - pInit->container = container; - pInit->pUserData = pUserData; - pInit->pUserDataMD = pUserDataMD; - - pInit->bs.onRead = onRead; - pInit->bs.onSeek = onSeek; - pInit->bs.pUserData = pUserData; - drflac__reset_cache(&pInit->bs); - - - /* If the container is explicitly defined then we can try opening in relaxed mode. */ - relaxed = container != drflac_container_unknown; - - /* Skip over any ID3 tags. */ - for (;;) - { - if (onRead(pUserData, id, 4) != 4) - return DRFLAC_FALSE; /* Ran out of data. */ - pInit->runningFilePos += 4; - - if (id[0] == 'I' && id[1] == 'D' && id[2] == '3') - { - drflac_uint8 flags; - drflac_uint32 headerSize; - drflac_uint8 header[6]; - - if (onRead(pUserData, header, 6) != 6) - return DRFLAC_FALSE; /* Ran out of data. */ - pInit->runningFilePos += 6; - - flags = header[1]; - drflac_copy_memory(&headerSize, header+2, 4); - headerSize = drflac__unsynchsafe_32(drflac__be2host_32(headerSize)); - if (flags & 0x10) - headerSize += 10; - - if (!onSeek(pUserData, headerSize, drflac_seek_origin_current)) - return DRFLAC_FALSE; /* Failed to seek past the tag. */ - pInit->runningFilePos += headerSize; - } - else - break; - } - - if (id[0] == 'f' && id[1] == 'L' && id[2] == 'a' && id[3] == 'C') { - return drflac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); - } -#ifndef DR_FLAC_NO_OGG - if (id[0] == 'O' && id[1] == 'g' && id[2] == 'g' && id[3] == 'S') { - return drflac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); - } -#endif - - /* If we get here it means we likely don't have a header. Try opening in relaxed mode, if applicable. */ - if (relaxed) - { - if (container == drflac_container_native) - return drflac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); -#ifndef DR_FLAC_NO_OGG - if (container == drflac_container_ogg) - return drflac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); -#endif - } - - /* Unsupported container. */ - return DRFLAC_FALSE; -} - -void drflac__init_from_info(drflac* pFlac, drflac_init_info* pInit) -{ - drflac_assert(pFlac != NULL); - drflac_assert(pInit != NULL); - - drflac_zero_memory(pFlac, sizeof(*pFlac)); - pFlac->bs = pInit->bs; - pFlac->onMeta = pInit->onMeta; - pFlac->pUserDataMD = pInit->pUserDataMD; - pFlac->maxBlockSize = pInit->maxBlockSize; - pFlac->sampleRate = pInit->sampleRate; - pFlac->channels = (drflac_uint8)pInit->channels; - pFlac->bitsPerSample = (drflac_uint8)pInit->bitsPerSample; - pFlac->totalSampleCount = pInit->totalSampleCount; - pFlac->container = pInit->container; -} - -drflac* drflac_open_with_metadata_private(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, void* pUserDataMD) -{ - drflac_init_info init; - drflac* pFlac; - drflac_uint32 decodedSamplesAllocationSize; - drflac_uint32 allocationSize; - drflac_uint32 wholeSIMDVectorCountPerChannel; - -#ifndef DRFLAC_NO_CPUID - /* CPU support first. */ - drflac__init_cpu_caps(); -#endif - - if (!drflac__init_private(&init, onRead, onSeek, onMeta, container, pUserData, pUserDataMD)) - return NULL; - - /* The size of the allocation for the drflac object needs to be large enough to fit the following: - * 1) The main members of the drflac structure - * 2) A block of memory large enough to store the decoded samples of the largest frame in the stream - * 3) If the container is Ogg, a drflac_oggbs object - * - * The complicated part of the allocation is making sure there's enough room the decoded samples, taking into consideration - * the different SIMD instruction sets. - */ - allocationSize = sizeof(drflac); - - /* The allocation size for decoded frames depends on the number of 32-bit integers that fit inside the largest SIMD vector - * we are supporting. */ - if ((init.maxBlockSize % (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))) == 0) { - wholeSIMDVectorCountPerChannel = (init.maxBlockSize / (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))); - } else { - wholeSIMDVectorCountPerChannel = (init.maxBlockSize / (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))) + 1; - } - - decodedSamplesAllocationSize = wholeSIMDVectorCountPerChannel * DRFLAC_MAX_SIMD_VECTOR_SIZE * init.channels; - - allocationSize += decodedSamplesAllocationSize; - allocationSize += DRFLAC_MAX_SIMD_VECTOR_SIZE; /* Allocate extra bytes to ensure we have enough for alignment. */ - -#ifndef DR_FLAC_NO_OGG - /* There's additional data required for Ogg streams. */ - if (init.container == drflac_container_ogg) - allocationSize += sizeof(drflac_oggbs); -#endif - - pFlac = (drflac*)DRFLAC_MALLOC(allocationSize); - drflac__init_from_info(pFlac, &init); - pFlac->pDecodedSamples = (drflac_int32*)drflac_align((size_t)pFlac->pExtraData, DRFLAC_MAX_SIMD_VECTOR_SIZE); - -#ifndef DR_FLAC_NO_OGG - if (init.container == drflac_container_ogg) { - drflac_oggbs* oggbs = (drflac_oggbs*)((drflac_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize); - oggbs->onRead = onRead; - oggbs->onSeek = onSeek; - oggbs->pUserData = pUserData; - oggbs->currentBytePos = init.oggFirstBytePos; - oggbs->firstBytePos = init.oggFirstBytePos; - oggbs->serialNumber = init.oggSerial; - oggbs->bosPageHeader = init.oggBosHeader; - oggbs->bytesRemainingInPage = 0; - - /* The Ogg bistream needs to be layered on top of the original bitstream. */ - pFlac->bs.onRead = drflac__on_read_ogg; - pFlac->bs.onSeek = drflac__on_seek_ogg; - pFlac->bs.pUserData = (void*)oggbs; - pFlac->_oggbs = (void*)oggbs; - } -#endif - - /* Decode metadata before returning. */ - if (init.hasMetadataBlocks) - { - if (!drflac__read_and_decode_metadata(pFlac)) - { - DRFLAC_FREE(pFlac); - return NULL; - } - } - - /* If we get here, but don't have a STREAMINFO block, it means we've opened the stream in relaxed mode and need to decode - * the first frame. */ - if (!init.hasStreamInfoBlock) - { - pFlac->currentFrame.header = init.firstFrameHeader; - do - { - drflac_result result = drflac__decode_frame(pFlac); - if (result == DRFLAC_SUCCESS) - break; - - if (result == DRFLAC_CRC_MISMATCH) - { - if (drflac__read_next_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFrame.header)) - continue; - } - DRFLAC_FREE(pFlac); - return NULL; - } while (1); - } - - return pFlac; -} - -#ifndef DR_FLAC_NO_STDIO -typedef void* drflac_file; - -#if defined(DR_FLAC_NO_WIN32_IO) || !defined(_WIN32) -#include - -static size_t drflac__on_read_stdio(void* pUserData, void* bufferOut, size_t bytesToRead) -{ - return fread(bufferOut, 1, bytesToRead, (FILE*)pUserData); -} - -static drflac_bool32 drflac__on_seek_stdio(void* pUserData, int offset, drflac_seek_origin origin) -{ - drflac_assert(offset > 0 || (offset == 0 && origin == drflac_seek_origin_start)); - - return fseek((FILE*)pUserData, offset, (origin == drflac_seek_origin_current) ? SEEK_CUR : SEEK_SET) == 0; -} - -static drflac_file drflac__open_file_handle(const char* filename) -{ - FILE* pFile; -#ifdef _MSC_VER - if (fopen_s(&pFile, filename, "rb") != 0) { - return NULL; - } -#else - pFile = fopen(filename, "rb"); - if (pFile == NULL) { - return NULL; - } -#endif - - return (drflac_file)pFile; -} - -static void drflac__close_file_handle(drflac_file file) -{ - fclose((FILE*)file); -} -#else -#include - -/* This doesn't seem to be defined for VC6. */ -#ifndef INVALID_SET_FILE_POINTER -#define INVALID_SET_FILE_POINTER ((DWORD)-1) -#endif - -static size_t drflac__on_read_stdio(void* pUserData, void* bufferOut, size_t bytesToRead) -{ - DWORD bytesRead; - - drflac_assert(bytesToRead < 0xFFFFFFFF); /* dr_flac will never request huge amounts of data at a time. This is a safe assertion. */ - - ReadFile((HANDLE)pUserData, bufferOut, (DWORD)bytesToRead, &bytesRead, NULL); - - return (size_t)bytesRead; -} - -static drflac_bool32 drflac__on_seek_stdio(void* pUserData, int offset, drflac_seek_origin origin) -{ - drflac_assert(offset > 0 || (offset == 0 && origin == drflac_seek_origin_start)); - - return SetFilePointer((HANDLE)pUserData, offset, NULL, (origin == drflac_seek_origin_current) ? FILE_CURRENT : FILE_BEGIN) != INVALID_SET_FILE_POINTER; -} - -static drflac_file drflac__open_file_handle(const char* filename) -{ - HANDLE hFile = CreateFileA(filename, FILE_GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (hFile == INVALID_HANDLE_VALUE) { - return NULL; - } - - return (drflac_file)hFile; -} - -static void drflac__close_file_handle(drflac_file file) -{ - CloseHandle((HANDLE)file); -} -#endif - - -drflac* drflac_open_file(const char* filename) -{ - drflac_file file = drflac__open_file_handle(filename); - if (file == NULL) { - return NULL; - } - - drflac* pFlac = drflac_open(drflac__on_read_stdio, drflac__on_seek_stdio, (void*)file); - if (pFlac == NULL) { - drflac__close_file_handle(file); - return NULL; - } - - return pFlac; -} - -drflac* drflac_open_file_with_metadata(const char* filename, drflac_meta_proc onMeta, void* pUserData) -{ - drflac_file file = drflac__open_file_handle(filename); - if (file == NULL) { - return NULL; - } - - drflac* pFlac = drflac_open_with_metadata_private(drflac__on_read_stdio, drflac__on_seek_stdio, onMeta, drflac_container_unknown, (void*)file, pUserData); - if (pFlac == NULL) { - drflac__close_file_handle(file); - return pFlac; - } - - return pFlac; -} -#endif /* DR_FLAC_NO_STDIO */ - -static size_t drflac__on_read_memory(void* pUserData, void* bufferOut, size_t bytesToRead) -{ - size_t bytesRemaining; - drflac__memory_stream* memoryStream = (drflac__memory_stream*)pUserData; - drflac_assert(memoryStream != NULL); - drflac_assert(memoryStream->dataSize >= memoryStream->currentReadPos); - - bytesRemaining = memoryStream->dataSize - memoryStream->currentReadPos; - if (bytesToRead > bytesRemaining) - bytesToRead = bytesRemaining; - - if (bytesToRead > 0) - { - drflac_copy_memory(bufferOut, memoryStream->data + memoryStream->currentReadPos, bytesToRead); - memoryStream->currentReadPos += bytesToRead; - } - - return bytesToRead; -} - -static drflac_bool32 drflac__on_seek_memory(void* pUserData, int offset, drflac_seek_origin origin) -{ - drflac__memory_stream* memoryStream = (drflac__memory_stream*)pUserData; - drflac_assert(memoryStream != NULL); - drflac_assert(offset > 0 || (offset == 0 && origin == drflac_seek_origin_start)); - - if (origin == drflac_seek_origin_current) - { - if (memoryStream->currentReadPos + offset <= memoryStream->dataSize) - memoryStream->currentReadPos += offset; - else - memoryStream->currentReadPos = memoryStream->dataSize; /* Trying to seek too far forward. */ - } else { - if ((drflac_uint32)offset <= memoryStream->dataSize) { - memoryStream->currentReadPos = offset; - } else { - memoryStream->currentReadPos = memoryStream->dataSize; /* Trying to seek too far forward. */ - } - } - - return DRFLAC_TRUE; -} - -drflac* drflac_open_memory(const void* data, size_t dataSize) -{ - drflac__memory_stream memoryStream; - drflac* pFlac; - - memoryStream.data = (const unsigned char*)data; - memoryStream.dataSize = dataSize; - memoryStream.currentReadPos = 0; - pFlac = drflac_open(drflac__on_read_memory, drflac__on_seek_memory, &memoryStream); - if (pFlac == NULL) - return NULL; - - pFlac->memoryStream = memoryStream; - - /* This is an awful hack... */ -#ifndef DR_FLAC_NO_OGG - if (pFlac->container == drflac_container_ogg) - { - drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; - oggbs->pUserData = &pFlac->memoryStream; - } - else -#endif - { - pFlac->bs.pUserData = &pFlac->memoryStream; - } - - return pFlac; -} - -drflac* drflac_open_memory_with_metadata(const void* data, size_t dataSize, drflac_meta_proc onMeta, void* pUserData) -{ - drflac__memory_stream memoryStream; - drflac* pFlac; - - memoryStream.data = (const unsigned char*)data; - memoryStream.dataSize = dataSize; - memoryStream.currentReadPos = 0; - - pFlac = drflac_open_with_metadata_private(drflac__on_read_memory, drflac__on_seek_memory, onMeta, drflac_container_unknown, &memoryStream, pUserData); - if (pFlac == NULL) - return NULL; - - pFlac->memoryStream = memoryStream; - - /* This is an awful hack... */ -#ifndef DR_FLAC_NO_OGG - if (pFlac->container == drflac_container_ogg) - { - drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; - oggbs->pUserData = &pFlac->memoryStream; - } - else -#endif - { - pFlac->bs.pUserData = &pFlac->memoryStream; - } - - return pFlac; -} - - - -drflac* drflac_open(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData) -{ - return drflac_open_with_metadata_private(onRead, onSeek, NULL, drflac_container_unknown, pUserData, pUserData); -} -drflac* drflac_open_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_container container, void* pUserData) -{ - return drflac_open_with_metadata_private(onRead, onSeek, NULL, container, pUserData, pUserData); -} - -drflac* drflac_open_with_metadata(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData) -{ - return drflac_open_with_metadata_private(onRead, onSeek, onMeta, drflac_container_unknown, pUserData, pUserData); -} -drflac* drflac_open_with_metadata_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData) -{ - return drflac_open_with_metadata_private(onRead, onSeek, onMeta, container, pUserData, pUserData); -} - -void drflac_close(drflac* pFlac) -{ - if (pFlac == NULL) { - return; - } - -#ifndef DR_FLAC_NO_STDIO - /* If we opened the file with drflac_open_file() we will want to close the file handle. We can know whether or not drflac_open_file() - * was used by looking at the callbacks. */ - if (pFlac->bs.onRead == drflac__on_read_stdio) - drflac__close_file_handle((drflac_file)pFlac->bs.pUserData); - -#ifndef DR_FLAC_NO_OGG - /* Need to clean up Ogg streams a bit differently due to the way the bit streaming is chained. */ - if (pFlac->container == drflac_container_ogg) - { - drflac_assert(pFlac->bs.onRead == drflac__on_read_ogg); - drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; - if (oggbs->onRead == drflac__on_read_stdio) - drflac__close_file_handle((drflac_file)oggbs->pUserData); - } -#endif -#endif - - DRFLAC_FREE(pFlac); -} - -drflac_uint64 drflac__read_s32__misaligned(drflac* pFlac, drflac_uint64 samplesToRead, drflac_int32* bufferOut) -{ - drflac_uint64 samplesRead = 0; - unsigned int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFrame.header.channelAssignment); - - /* We should never be calling this when the number of samples to read is >= the sample count. */ - drflac_assert(samplesToRead < channelCount); - drflac_assert(pFlac->currentFrame.samplesRemaining > 0 && samplesToRead <= pFlac->currentFrame.samplesRemaining); - - while (samplesToRead > 0) - { - drflac_uint64 totalSamplesInFrame = pFlac->currentFrame.header.blockSize * channelCount; - drflac_uint64 samplesReadFromFrameSoFar = totalSamplesInFrame - pFlac->currentFrame.samplesRemaining; - drflac_uint64 channelIndex = samplesReadFromFrameSoFar % channelCount; - - drflac_uint64 nextSampleInFrame = samplesReadFromFrameSoFar / channelCount; - - int decodedSample = 0; - switch (pFlac->currentFrame.header.channelAssignment) - { - case DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: - { - if (channelIndex == 0) { - decodedSample = pFlac->currentFrame.subframes[channelIndex].pDecodedSamples[nextSampleInFrame]; - } else { - int side = pFlac->currentFrame.subframes[channelIndex + 0].pDecodedSamples[nextSampleInFrame]; - int left = pFlac->currentFrame.subframes[channelIndex - 1].pDecodedSamples[nextSampleInFrame]; - decodedSample = left - side; - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: - { - if (channelIndex == 0) { - int side = pFlac->currentFrame.subframes[channelIndex + 0].pDecodedSamples[nextSampleInFrame]; - int right = pFlac->currentFrame.subframes[channelIndex + 1].pDecodedSamples[nextSampleInFrame]; - decodedSample = side + right; - } else { - decodedSample = pFlac->currentFrame.subframes[channelIndex].pDecodedSamples[nextSampleInFrame]; - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE: - { - int mid; - int side; - if (channelIndex == 0) { - mid = pFlac->currentFrame.subframes[channelIndex + 0].pDecodedSamples[nextSampleInFrame]; - side = pFlac->currentFrame.subframes[channelIndex + 1].pDecodedSamples[nextSampleInFrame]; - - mid = (((unsigned int)mid) << 1) | (side & 0x01); - decodedSample = (mid + side) >> 1; - } else { - mid = pFlac->currentFrame.subframes[channelIndex - 1].pDecodedSamples[nextSampleInFrame]; - side = pFlac->currentFrame.subframes[channelIndex + 0].pDecodedSamples[nextSampleInFrame]; - - mid = (((unsigned int)mid) << 1) | (side & 0x01); - decodedSample = (mid - side) >> 1; - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: - default: - { - decodedSample = pFlac->currentFrame.subframes[channelIndex].pDecodedSamples[nextSampleInFrame]; - } break; - } - - - decodedSample <<= ((32 - pFlac->bitsPerSample) + pFlac->currentFrame.subframes[channelIndex].wastedBitsPerSample); - - if (bufferOut) { - *bufferOut++ = decodedSample; - } - - samplesRead += 1; - pFlac->currentFrame.samplesRemaining -= 1; - samplesToRead -= 1; - } - - return samplesRead; -} - -drflac_uint64 drflac__seek_forward_by_samples(drflac* pFlac, drflac_uint64 samplesToRead) -{ - drflac_uint64 samplesRead = 0; - while (samplesToRead > 0) { - if (pFlac->currentFrame.samplesRemaining == 0) - { - if (!drflac__read_and_decode_next_frame(pFlac)) - break; /* Couldn't read the next frame, so just break from the loop and return. */ - } - else - { - samplesRead += 1; - pFlac->currentFrame.samplesRemaining -= 1; - samplesToRead -= 1; - } - } - - return samplesRead; -} - -drflac_uint64 drflac_read_s32(drflac* pFlac, drflac_uint64 samplesToRead, drflac_int32* bufferOut) -{ - drflac_uint64 samplesRead = 0; - /* Note that is allowed to be null, in which case this will be treated as something like a seek. */ - if (pFlac == NULL || samplesToRead == 0) - return 0; - - if (bufferOut == NULL) - return drflac__seek_forward_by_samples(pFlac, samplesToRead); - - while (samplesToRead > 0) - { - /* If we've run out of samples in this frame, go to the next. */ - if (pFlac->currentFrame.samplesRemaining == 0) - { - if (!drflac__read_and_decode_next_frame(pFlac)) - break; /* Couldn't read the next frame, so just break from the loop and return. */ - } - else - { - drflac_uint64 alignedSamplesRead; - drflac_uint64 alignedSampleCountPerChannel; - drflac_uint64 firstAlignedSampleInFrame; - unsigned int unusedBitsPerSample; - /* Here is where we grab the samples and interleave them. */ - unsigned int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFrame.header.channelAssignment); - drflac_uint64 totalSamplesInFrame = pFlac->currentFrame.header.blockSize * channelCount; - drflac_uint64 samplesReadFromFrameSoFar = totalSamplesInFrame - pFlac->currentFrame.samplesRemaining; - drflac_uint64 misalignedSampleCount = samplesReadFromFrameSoFar % channelCount; - if (misalignedSampleCount > 0) - { - drflac_uint64 misalignedSamplesRead = drflac__read_s32__misaligned(pFlac, misalignedSampleCount, bufferOut); - samplesRead += misalignedSamplesRead; - samplesReadFromFrameSoFar += misalignedSamplesRead; - bufferOut += misalignedSamplesRead; - samplesToRead -= misalignedSamplesRead; - } - - alignedSampleCountPerChannel = samplesToRead / channelCount; - if (alignedSampleCountPerChannel > pFlac->currentFrame.samplesRemaining / channelCount) - alignedSampleCountPerChannel = pFlac->currentFrame.samplesRemaining / channelCount; - - firstAlignedSampleInFrame = samplesReadFromFrameSoFar / channelCount; - unusedBitsPerSample = 32 - pFlac->bitsPerSample; - - switch (pFlac->currentFrame.header.channelAssignment) - { - case DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: - { - drflac_uint64 i; - const drflac_int32* pDecodedSamples0 = pFlac->currentFrame.subframes[0].pDecodedSamples + firstAlignedSampleInFrame; - const drflac_int32* pDecodedSamples1 = pFlac->currentFrame.subframes[1].pDecodedSamples + firstAlignedSampleInFrame; - - for (i = 0; i < alignedSampleCountPerChannel; ++i) { - int left = pDecodedSamples0[i]; - int side = pDecodedSamples1[i]; - int right = left - side; - - bufferOut[i*2+0] = left << (unusedBitsPerSample + pFlac->currentFrame.subframes[0].wastedBitsPerSample); - bufferOut[i*2+1] = right << (unusedBitsPerSample + pFlac->currentFrame.subframes[1].wastedBitsPerSample); - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: - { - drflac_uint64 i; - const drflac_int32* pDecodedSamples0 = pFlac->currentFrame.subframes[0].pDecodedSamples + firstAlignedSampleInFrame; - const drflac_int32* pDecodedSamples1 = pFlac->currentFrame.subframes[1].pDecodedSamples + firstAlignedSampleInFrame; - - for (i = 0; i < alignedSampleCountPerChannel; ++i) { - int side = pDecodedSamples0[i]; - int right = pDecodedSamples1[i]; - int left = right + side; - - bufferOut[i*2+0] = left << (unusedBitsPerSample + pFlac->currentFrame.subframes[0].wastedBitsPerSample); - bufferOut[i*2+1] = right << (unusedBitsPerSample + pFlac->currentFrame.subframes[1].wastedBitsPerSample); - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE: - { - drflac_uint64 i; - const drflac_int32* pDecodedSamples0 = pFlac->currentFrame.subframes[0].pDecodedSamples + firstAlignedSampleInFrame; - const drflac_int32* pDecodedSamples1 = pFlac->currentFrame.subframes[1].pDecodedSamples + firstAlignedSampleInFrame; - - for (i = 0; i < alignedSampleCountPerChannel; ++i) { - int side = pDecodedSamples1[i]; - int mid = (((drflac_uint32)pDecodedSamples0[i]) << 1) | (side & 0x01); - - bufferOut[i*2+0] = ((mid + side) >> 1) << (unusedBitsPerSample + pFlac->currentFrame.subframes[0].wastedBitsPerSample); - bufferOut[i*2+1] = ((mid - side) >> 1) << (unusedBitsPerSample + pFlac->currentFrame.subframes[1].wastedBitsPerSample); - } - } break; - - case DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: - default: - { - if (pFlac->currentFrame.header.channelAssignment == 1) /* 1 = Stereo */ - { - drflac_uint64 i; - /* Stereo optimized inner loop unroll. */ - const drflac_int32* pDecodedSamples0 = pFlac->currentFrame.subframes[0].pDecodedSamples + firstAlignedSampleInFrame; - const drflac_int32* pDecodedSamples1 = pFlac->currentFrame.subframes[1].pDecodedSamples + firstAlignedSampleInFrame; - - for (i = 0; i < alignedSampleCountPerChannel; ++i) { - bufferOut[i*2+0] = pDecodedSamples0[i] << (unusedBitsPerSample + pFlac->currentFrame.subframes[0].wastedBitsPerSample); - bufferOut[i*2+1] = pDecodedSamples1[i] << (unusedBitsPerSample + pFlac->currentFrame.subframes[1].wastedBitsPerSample); - } - } - else - { - drflac_uint64 i; - - /* Generic interleaving. */ - for (i = 0; i < alignedSampleCountPerChannel; ++i) - { - unsigned j; - for (j = 0; j < channelCount; ++j) - { - bufferOut[(i*channelCount)+j] = (pFlac->currentFrame.subframes[j].pDecodedSamples[firstAlignedSampleInFrame + i]) << (unusedBitsPerSample + pFlac->currentFrame.subframes[j].wastedBitsPerSample); - } - } - } - } break; - } - - alignedSamplesRead = alignedSampleCountPerChannel * channelCount; - samplesRead += alignedSamplesRead; - bufferOut += alignedSamplesRead; - samplesToRead -= alignedSamplesRead; - pFlac->currentFrame.samplesRemaining -= (unsigned int)alignedSamplesRead; - - /* At this point we may still have some excess samples left to read. */ - if (samplesToRead > 0 && pFlac->currentFrame.samplesRemaining > 0) { - drflac_uint64 excessSamplesRead = 0; - if (samplesToRead < pFlac->currentFrame.samplesRemaining) { - excessSamplesRead = drflac__read_s32__misaligned(pFlac, samplesToRead, bufferOut); - } else { - excessSamplesRead = drflac__read_s32__misaligned(pFlac, pFlac->currentFrame.samplesRemaining, bufferOut); - } - - samplesRead += excessSamplesRead; - bufferOut += excessSamplesRead; - samplesToRead -= excessSamplesRead; - } - } - } - - return samplesRead; -} - -drflac_uint64 drflac_read_s16(drflac* pFlac, drflac_uint64 samplesToRead, drflac_int16* pBufferOut) -{ - /* This reads samples in 2 passes and can probably be optimized. */ - drflac_uint64 totalSamplesRead = 0; - - while (samplesToRead > 0) - { - drflac_uint64 i; - drflac_int32 samples32[4096]; - drflac_uint64 samplesJustRead = drflac_read_s32(pFlac, (samplesToRead > 4096) ? 4096 : samplesToRead, samples32); - if (samplesJustRead == 0) - break; /* Reached the end. */ - - /* s32 -> s16 */ - for (i = 0; i < samplesJustRead; ++i) - pBufferOut[i] = (drflac_int16)(samples32[i] >> 16); - - totalSamplesRead += samplesJustRead; - samplesToRead -= samplesJustRead; - pBufferOut += samplesJustRead; - } - - return totalSamplesRead; -} - -drflac_uint64 drflac_read_f32(drflac* pFlac, drflac_uint64 samplesToRead, float* pBufferOut) -{ - /* This reads samples in 2 passes and can probably be optimized. */ - drflac_uint64 totalSamplesRead = 0; - - while (samplesToRead > 0) - { - drflac_uint64 i; - drflac_int32 samples32[4096]; - drflac_uint64 samplesJustRead = drflac_read_s32(pFlac, (samplesToRead > 4096) ? 4096 : samplesToRead, samples32); - if (samplesJustRead == 0) - break; /* Reached the end. */ - - /* s32 -> f32 */ - for (i = 0; i < samplesJustRead; ++i) - pBufferOut[i] = (float)(samples32[i] / 2147483648.0); - - totalSamplesRead += samplesJustRead; - samplesToRead -= samplesJustRead; - pBufferOut += samplesJustRead; - } - - return totalSamplesRead; -} - -drflac_bool32 drflac_seek_to_sample(drflac* pFlac, drflac_uint64 sampleIndex) -{ - if (pFlac == NULL) - return DRFLAC_FALSE; - - /* If we don't know where the first frame begins then we can't seek. This will happen when the STREAMINFO block was not present - * when the decoder was opened. */ - if (pFlac->firstFramePos == 0) - return DRFLAC_FALSE; - - if (sampleIndex == 0) - return drflac__seek_to_first_frame(pFlac); - - /* Clamp the sample to the end. */ - if (sampleIndex >= pFlac->totalSampleCount) - sampleIndex = pFlac->totalSampleCount - 1; - - - /* Different techniques depending on encapsulation. Using the native FLAC seektable with Ogg encapsulation is a bit awkward so - * we'll instead use Ogg's natural seeking facility. */ -#ifndef DR_FLAC_NO_OGG - if (pFlac->container == drflac_container_ogg) - return drflac_ogg__seek_to_sample(pFlac, sampleIndex); - else -#endif - { - /* First try seeking via the seek table. If this fails, fall back to a brute force seek which is much slower. */ - if (!drflac__seek_to_sample__seek_table(pFlac, sampleIndex)) - return drflac__seek_to_sample__brute_force(pFlac, sampleIndex); - } - - - return DRFLAC_TRUE; -} - - - -/* High Level APIs */ - -/* I couldn't figure out where SIZE_MAX was defined for VC6. If anybody knows, let me know. */ -#if defined(_MSC_VER) && _MSC_VER <= 1200 -#ifdef DRFLAC_64BIT -#define SIZE_MAX ((drflac_uint64)0xFFFFFFFFFFFFFFFF) -#else -#define SIZE_MAX 0xFFFFFFFF -#endif -#endif - -/* Using a macro as the definition of the drflac__full_decode_and_close_*() API family. Sue me. */ -#define DRFLAC_DEFINE_FULL_DECODE_AND_CLOSE(extension, type) \ -static type* drflac__full_decode_and_close_ ## extension (drflac* pFlac, unsigned int* channelsOut, unsigned int* sampleRateOut, drflac_uint64* totalSampleCountOut)\ -{ \ - drflac_uint64 totalSampleCount; \ - type* pSampleData = NULL; \ - drflac_assert(pFlac != NULL); \ - totalSampleCount = pFlac->totalSampleCount; \ - if (totalSampleCount == 0) { \ - type buffer[4096]; \ - drflac_uint64 samplesRead; \ - size_t sampleDataBufferSize = sizeof(buffer); \ - pSampleData = (type*)DRFLAC_MALLOC(sampleDataBufferSize); \ - if (pSampleData == NULL) \ - goto on_error; \ - while ((samplesRead = (drflac_uint64)drflac_read_##extension(pFlac, sizeof(buffer)/sizeof(buffer[0]), buffer)) > 0) { \ - if (((totalSampleCount + samplesRead) * sizeof(type)) > sampleDataBufferSize) { \ - type *pNewSampleData; \ - sampleDataBufferSize *= 2; \ - pNewSampleData = (type*)DRFLAC_REALLOC(pSampleData, sampleDataBufferSize); \ - if (pNewSampleData == NULL) { \ - DRFLAC_FREE(pSampleData); \ - goto on_error; \ - } \ - \ - pSampleData = pNewSampleData; \ - } \ - \ - drflac_copy_memory(pSampleData + totalSampleCount, buffer, (size_t)(samplesRead*sizeof(type))); \ - totalSampleCount += samplesRead; \ - } \ - \ - /* At this point everything should be decoded, but we just want to fill the unused part buffer with silence - need to \ - protect those ears from random noise! */ \ - drflac_zero_memory(pSampleData + totalSampleCount, (size_t)(sampleDataBufferSize - totalSampleCount*sizeof(type))); \ - } else { \ - drflac_uint64 dataSize = totalSampleCount * sizeof(type); \ - if (dataSize > SIZE_MAX) { \ - goto on_error; /* The decoded data is too big. */ \ - } \ - \ - pSampleData = (type*)DRFLAC_MALLOC((size_t)dataSize); /* <-- Safe cast as per the check above. */ \ - if (pSampleData == NULL) { \ - goto on_error; \ - } \ - \ - totalSampleCount = drflac_read_##extension(pFlac, pFlac->totalSampleCount, pSampleData); \ - } \ - \ - if (sampleRateOut) *sampleRateOut = pFlac->sampleRate; \ - if (channelsOut) *channelsOut = pFlac->channels; \ - if (totalSampleCountOut) *totalSampleCountOut = totalSampleCount; \ - \ - drflac_close(pFlac); \ - return pSampleData; \ - \ -on_error: \ - drflac_close(pFlac); \ - return NULL; \ -} - -DRFLAC_DEFINE_FULL_DECODE_AND_CLOSE(s32, drflac_int32) -DRFLAC_DEFINE_FULL_DECODE_AND_CLOSE(s16, drflac_int16) -DRFLAC_DEFINE_FULL_DECODE_AND_CLOSE(f32, float) - -drflac_int32* drflac_open_and_decode_s32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - /* Safety. */ - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open(onRead, onSeek, pUserData); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s32(pFlac, channels, sampleRate, totalSampleCount); -} - -drflac_int16* drflac_open_and_decode_s16(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - /* Safety. */ - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open(onRead, onSeek, pUserData); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s16(pFlac, channels, sampleRate, totalSampleCount); -} - -float* drflac_open_and_decode_f32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - /* Safety. */ - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open(onRead, onSeek, pUserData); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_f32(pFlac, channels, sampleRate, totalSampleCount); -} - -#ifndef DR_FLAC_NO_STDIO -drflac_int32* drflac_open_and_decode_file_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_file(filename); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s32(pFlac, channels, sampleRate, totalSampleCount); -} - -drflac_int16* drflac_open_and_decode_file_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_file(filename); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s16(pFlac, channels, sampleRate, totalSampleCount); -} - -float* drflac_open_and_decode_file_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_file(filename); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_f32(pFlac, channels, sampleRate, totalSampleCount); -} -#endif - -drflac_int32* drflac_open_and_decode_memory_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac *pFlac; - - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_memory(data, dataSize); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s32(pFlac, channels, sampleRate, totalSampleCount); -} - -drflac_int16* drflac_open_and_decode_memory_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_memory(data, dataSize); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_s16(pFlac, channels, sampleRate, totalSampleCount); -} - -float* drflac_open_and_decode_memory_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalSampleCount) -{ - drflac* pFlac; - if (sampleRate) *sampleRate = 0; - if (channels) *channels = 0; - if (totalSampleCount) *totalSampleCount = 0; - - pFlac = drflac_open_memory(data, dataSize); - if (pFlac == NULL) - return NULL; - - return drflac__full_decode_and_close_f32(pFlac, channels, sampleRate, totalSampleCount); -} - -void drflac_free(void* pSampleDataReturnedByOpenAndDecode) -{ - DRFLAC_FREE(pSampleDataReturnedByOpenAndDecode); -} - - - - -void drflac_init_vorbis_comment_iterator(drflac_vorbis_comment_iterator* pIter, drflac_uint32 commentCount, const char* pComments) -{ - if (pIter == NULL) - return; - - pIter->countRemaining = commentCount; - pIter->pRunningData = pComments; -} - -const char* drflac_next_vorbis_comment(drflac_vorbis_comment_iterator* pIter, drflac_uint32* pCommentLengthOut) -{ - const char* pComment; - drflac_uint32 length; - /* Safety. */ - if (pCommentLengthOut) *pCommentLengthOut = 0; - - if (pIter == NULL || pIter->countRemaining == 0 || pIter->pRunningData == NULL) - return NULL; - - length = drflac__le2host_32(*(drflac_uint32*)pIter->pRunningData); - pIter->pRunningData += 4; - - pComment = pIter->pRunningData; - pIter->pRunningData += length; - pIter->countRemaining -= 1; - - if (pCommentLengthOut) *pCommentLengthOut = length; - return pComment; -} -#endif /* DR_FLAC_IMPLEMENTATION */ - -/* -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to -*/ +/* +FLAC audio decoder. Choice of public domain or MIT-0. See license statements at the end of this file. +dr_flac - v0.12.42 - 2023-11-02 + +David Reid - mackron@gmail.com + +GitHub: https://github.com/mackron/dr_libs +*/ + +/* +RELEASE NOTES - v0.12.0 +======================= +Version 0.12.0 has breaking API changes including changes to the existing API and the removal of deprecated APIs. + + +Improved Client-Defined Memory Allocation +----------------------------------------- +The main change with this release is the addition of a more flexible way of implementing custom memory allocation routines. The +existing system of DRFLAC_MALLOC, DRFLAC_REALLOC and DRFLAC_FREE are still in place and will be used by default when no custom +allocation callbacks are specified. + +To use the new system, you pass in a pointer to a drflac_allocation_callbacks object to drflac_open() and family, like this: + + void* my_malloc(size_t sz, void* pUserData) + { + return malloc(sz); + } + void* my_realloc(void* p, size_t sz, void* pUserData) + { + return realloc(p, sz); + } + void my_free(void* p, void* pUserData) + { + free(p); + } + + ... + + drflac_allocation_callbacks allocationCallbacks; + allocationCallbacks.pUserData = &myData; + allocationCallbacks.onMalloc = my_malloc; + allocationCallbacks.onRealloc = my_realloc; + allocationCallbacks.onFree = my_free; + drflac* pFlac = drflac_open_file("my_file.flac", &allocationCallbacks); + +The advantage of this new system is that it allows you to specify user data which will be passed in to the allocation routines. + +Passing in null for the allocation callbacks object will cause dr_flac to use defaults which is the same as DRFLAC_MALLOC, +DRFLAC_REALLOC and DRFLAC_FREE and the equivalent of how it worked in previous versions. + +Every API that opens a drflac object now takes this extra parameter. These include the following: + + drflac_open() + drflac_open_relaxed() + drflac_open_with_metadata() + drflac_open_with_metadata_relaxed() + drflac_open_file() + drflac_open_file_with_metadata() + drflac_open_memory() + drflac_open_memory_with_metadata() + drflac_open_and_read_pcm_frames_s32() + drflac_open_and_read_pcm_frames_s16() + drflac_open_and_read_pcm_frames_f32() + drflac_open_file_and_read_pcm_frames_s32() + drflac_open_file_and_read_pcm_frames_s16() + drflac_open_file_and_read_pcm_frames_f32() + drflac_open_memory_and_read_pcm_frames_s32() + drflac_open_memory_and_read_pcm_frames_s16() + drflac_open_memory_and_read_pcm_frames_f32() + + + +Optimizations +------------- +Seeking performance has been greatly improved. A new binary search based seeking algorithm has been introduced which significantly +improves performance over the brute force method which was used when no seek table was present. Seek table based seeking also takes +advantage of the new binary search seeking system to further improve performance there as well. Note that this depends on CRC which +means it will be disabled when DR_FLAC_NO_CRC is used. + +The SSE4.1 pipeline has been cleaned up and optimized. You should see some improvements with decoding speed of 24-bit files in +particular. 16-bit streams should also see some improvement. + +drflac_read_pcm_frames_s16() has been optimized. Previously this sat on top of drflac_read_pcm_frames_s32() and performed it's s32 +to s16 conversion in a second pass. This is now all done in a single pass. This includes SSE2 and ARM NEON optimized paths. + +A minor optimization has been implemented for drflac_read_pcm_frames_s32(). This will now use an SSE2 optimized pipeline for stereo +channel reconstruction which is the last part of the decoding process. + +The ARM build has seen a few improvements. The CLZ (count leading zeroes) and REV (byte swap) instructions are now used when +compiling with GCC and Clang which is achieved using inline assembly. The CLZ instruction requires ARM architecture version 5 at +compile time and the REV instruction requires ARM architecture version 6. + +An ARM NEON optimized pipeline has been implemented. To enable this you'll need to add -mfpu=neon to the command line when compiling. + + +Removed APIs +------------ +The following APIs were deprecated in version 0.11.0 and have been completely removed in version 0.12.0: + + drflac_read_s32() -> drflac_read_pcm_frames_s32() + drflac_read_s16() -> drflac_read_pcm_frames_s16() + drflac_read_f32() -> drflac_read_pcm_frames_f32() + drflac_seek_to_sample() -> drflac_seek_to_pcm_frame() + drflac_open_and_decode_s32() -> drflac_open_and_read_pcm_frames_s32() + drflac_open_and_decode_s16() -> drflac_open_and_read_pcm_frames_s16() + drflac_open_and_decode_f32() -> drflac_open_and_read_pcm_frames_f32() + drflac_open_and_decode_file_s32() -> drflac_open_file_and_read_pcm_frames_s32() + drflac_open_and_decode_file_s16() -> drflac_open_file_and_read_pcm_frames_s16() + drflac_open_and_decode_file_f32() -> drflac_open_file_and_read_pcm_frames_f32() + drflac_open_and_decode_memory_s32() -> drflac_open_memory_and_read_pcm_frames_s32() + drflac_open_and_decode_memory_s16() -> drflac_open_memory_and_read_pcm_frames_s16() + drflac_open_and_decode_memory_f32() -> drflac_open_memroy_and_read_pcm_frames_f32() + +Prior versions of dr_flac operated on a per-sample basis whereas now it operates on PCM frames. The removed APIs all relate +to the old per-sample APIs. You now need to use the "pcm_frame" versions. +*/ + + +/* +Introduction +============ +dr_flac is a single file library. To use it, do something like the following in one .c file. + + ```c + #define DR_FLAC_IMPLEMENTATION + #include "dr_flac.h" + ``` + +You can then #include this file in other parts of the program as you would with any other header file. To decode audio data, do something like the following: + + ```c + drflac* pFlac = drflac_open_file("MySong.flac", NULL); + if (pFlac == NULL) { + // Failed to open FLAC file + } + + drflac_int32* pSamples = malloc(pFlac->totalPCMFrameCount * pFlac->channels * sizeof(drflac_int32)); + drflac_uint64 numberOfInterleavedSamplesActuallyRead = drflac_read_pcm_frames_s32(pFlac, pFlac->totalPCMFrameCount, pSamples); + ``` + +The drflac object represents the decoder. It is a transparent type so all the information you need, such as the number of channels and the bits per sample, +should be directly accessible - just make sure you don't change their values. Samples are always output as interleaved signed 32-bit PCM. In the example above +a native FLAC stream was opened, however dr_flac has seamless support for Ogg encapsulated FLAC streams as well. + +You do not need to decode the entire stream in one go - you just specify how many samples you'd like at any given time and the decoder will give you as many +samples as it can, up to the amount requested. Later on when you need the next batch of samples, just call it again. Example: + + ```c + while (drflac_read_pcm_frames_s32(pFlac, chunkSizeInPCMFrames, pChunkSamples) > 0) { + do_something(); + } + ``` + +You can seek to a specific PCM frame with `drflac_seek_to_pcm_frame()`. + +If you just want to quickly decode an entire FLAC file in one go you can do something like this: + + ```c + unsigned int channels; + unsigned int sampleRate; + drflac_uint64 totalPCMFrameCount; + drflac_int32* pSampleData = drflac_open_file_and_read_pcm_frames_s32("MySong.flac", &channels, &sampleRate, &totalPCMFrameCount, NULL); + if (pSampleData == NULL) { + // Failed to open and decode FLAC file. + } + + ... + + drflac_free(pSampleData, NULL); + ``` + +You can read samples as signed 16-bit integer and 32-bit floating-point PCM with the *_s16() and *_f32() family of APIs respectively, but note that these +should be considered lossy. + + +If you need access to metadata (album art, etc.), use `drflac_open_with_metadata()`, `drflac_open_file_with_metdata()` or `drflac_open_memory_with_metadata()`. +The rationale for keeping these APIs separate is that they're slightly slower than the normal versions and also just a little bit harder to use. dr_flac +reports metadata to the application through the use of a callback, and every metadata block is reported before `drflac_open_with_metdata()` returns. + +The main opening APIs (`drflac_open()`, etc.) will fail if the header is not present. The presents a problem in certain scenarios such as broadcast style +streams or internet radio where the header may not be present because the user has started playback mid-stream. To handle this, use the relaxed APIs: + + `drflac_open_relaxed()` + `drflac_open_with_metadata_relaxed()` + +It is not recommended to use these APIs for file based streams because a missing header would usually indicate a corrupt or perverse file. In addition, these +APIs can take a long time to initialize because they may need to spend a lot of time finding the first frame. + + + +Build Options +============= +#define these options before including this file. + +#define DR_FLAC_NO_STDIO + Disable `drflac_open_file()` and family. + +#define DR_FLAC_NO_OGG + Disables support for Ogg/FLAC streams. + +#define DR_FLAC_BUFFER_SIZE + Defines the size of the internal buffer to store data from onRead(). This buffer is used to reduce the number of calls back to the client for more data. + Larger values means more memory, but better performance. My tests show diminishing returns after about 4KB (which is the default). Consider reducing this if + you have a very efficient implementation of onRead(), or increase it if it's very inefficient. Must be a multiple of 8. + +#define DR_FLAC_NO_CRC + Disables CRC checks. This will offer a performance boost when CRC is unnecessary. This will disable binary search seeking. When seeking, the seek table will + be used if available. Otherwise the seek will be performed using brute force. + +#define DR_FLAC_NO_SIMD + Disables SIMD optimizations (SSE on x86/x64 architectures, NEON on ARM architectures). Use this if you are having compatibility issues with your compiler. + +#define DR_FLAC_NO_WCHAR + Disables all functions ending with `_w`. Use this if your compiler does not provide wchar.h. Not required if DR_FLAC_NO_STDIO is also defined. + + + +Notes +===== +- dr_flac does not support changing the sample rate nor channel count mid stream. +- dr_flac is not thread-safe, but its APIs can be called from any thread so long as you do your own synchronization. +- When using Ogg encapsulation, a corrupted metadata block will result in `drflac_open_with_metadata()` and `drflac_open()` returning inconsistent samples due + to differences in corrupted stream recorvery logic between the two APIs. +*/ + +#ifndef dr_flac_h +#define dr_flac_h + +#ifdef __cplusplus +extern "C" { +#endif + +#define DRFLAC_STRINGIFY(x) #x +#define DRFLAC_XSTRINGIFY(x) DRFLAC_STRINGIFY(x) + +#define DRFLAC_VERSION_MAJOR 0 +#define DRFLAC_VERSION_MINOR 12 +#define DRFLAC_VERSION_REVISION 42 +#define DRFLAC_VERSION_STRING DRFLAC_XSTRINGIFY(DRFLAC_VERSION_MAJOR) "." DRFLAC_XSTRINGIFY(DRFLAC_VERSION_MINOR) "." DRFLAC_XSTRINGIFY(DRFLAC_VERSION_REVISION) + +#include /* For size_t. */ + +/* Sized Types */ +typedef signed char drflac_int8; +typedef unsigned char drflac_uint8; +typedef signed short drflac_int16; +typedef unsigned short drflac_uint16; +typedef signed int drflac_int32; +typedef unsigned int drflac_uint32; +#if defined(_MSC_VER) && !defined(__clang__) + typedef signed __int64 drflac_int64; + typedef unsigned __int64 drflac_uint64; +#else + #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wlong-long" + #if defined(__clang__) + #pragma GCC diagnostic ignored "-Wc++11-long-long" + #endif + #endif + typedef signed long long drflac_int64; + typedef unsigned long long drflac_uint64; + #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))) + #pragma GCC diagnostic pop + #endif +#endif +#if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__)) || defined(_M_X64) || defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(_M_ARM64) || defined(__powerpc64__) + typedef drflac_uint64 drflac_uintptr; +#else + typedef drflac_uint32 drflac_uintptr; +#endif +typedef drflac_uint8 drflac_bool8; +typedef drflac_uint32 drflac_bool32; +#define DRFLAC_TRUE 1 +#define DRFLAC_FALSE 0 +/* End Sized Types */ + +/* Decorations */ +#if !defined(DRFLAC_API) + #if defined(DRFLAC_DLL) + #if defined(_WIN32) + #define DRFLAC_DLL_IMPORT __declspec(dllimport) + #define DRFLAC_DLL_EXPORT __declspec(dllexport) + #define DRFLAC_DLL_PRIVATE static + #else + #if defined(__GNUC__) && __GNUC__ >= 4 + #define DRFLAC_DLL_IMPORT __attribute__((visibility("default"))) + #define DRFLAC_DLL_EXPORT __attribute__((visibility("default"))) + #define DRFLAC_DLL_PRIVATE __attribute__((visibility("hidden"))) + #else + #define DRFLAC_DLL_IMPORT + #define DRFLAC_DLL_EXPORT + #define DRFLAC_DLL_PRIVATE static + #endif + #endif + + #if defined(DR_FLAC_IMPLEMENTATION) || defined(DRFLAC_IMPLEMENTATION) + #define DRFLAC_API DRFLAC_DLL_EXPORT + #else + #define DRFLAC_API DRFLAC_DLL_IMPORT + #endif + #define DRFLAC_PRIVATE DRFLAC_DLL_PRIVATE + #else + #define DRFLAC_API extern + #define DRFLAC_PRIVATE static + #endif +#endif +/* End Decorations */ + +#if defined(_MSC_VER) && _MSC_VER >= 1700 /* Visual Studio 2012 */ + #define DRFLAC_DEPRECATED __declspec(deprecated) +#elif (defined(__GNUC__) && __GNUC__ >= 4) /* GCC 4 */ + #define DRFLAC_DEPRECATED __attribute__((deprecated)) +#elif defined(__has_feature) /* Clang */ + #if __has_feature(attribute_deprecated) + #define DRFLAC_DEPRECATED __attribute__((deprecated)) + #else + #define DRFLAC_DEPRECATED + #endif +#else + #define DRFLAC_DEPRECATED +#endif + +DRFLAC_API void drflac_version(drflac_uint32* pMajor, drflac_uint32* pMinor, drflac_uint32* pRevision); +DRFLAC_API const char* drflac_version_string(void); + +/* Allocation Callbacks */ +typedef struct +{ + void* pUserData; + void* (* onMalloc)(size_t sz, void* pUserData); + void* (* onRealloc)(void* p, size_t sz, void* pUserData); + void (* onFree)(void* p, void* pUserData); +} drflac_allocation_callbacks; +/* End Allocation Callbacks */ + +/* +As data is read from the client it is placed into an internal buffer for fast access. This controls the size of that buffer. Larger values means more speed, +but also more memory. In my testing there is diminishing returns after about 4KB, but you can fiddle with this to suit your own needs. Must be a multiple of 8. +*/ +#ifndef DR_FLAC_BUFFER_SIZE +#define DR_FLAC_BUFFER_SIZE 4096 +#endif + + +/* Architecture Detection */ +#if defined(_WIN64) || defined(_LP64) || defined(__LP64__) +#define DRFLAC_64BIT +#endif + +#if defined(__x86_64__) || defined(_M_X64) + #define DRFLAC_X64 +#elif defined(__i386) || defined(_M_IX86) + #define DRFLAC_X86 +#elif defined(__arm__) || defined(_M_ARM) || defined(__arm64) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64) + #define DRFLAC_ARM +#endif +/* End Architecture Detection */ + + +#ifdef DRFLAC_64BIT +typedef drflac_uint64 drflac_cache_t; +#else +typedef drflac_uint32 drflac_cache_t; +#endif + +/* The various metadata block types. */ +#define DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO 0 +#define DRFLAC_METADATA_BLOCK_TYPE_PADDING 1 +#define DRFLAC_METADATA_BLOCK_TYPE_APPLICATION 2 +#define DRFLAC_METADATA_BLOCK_TYPE_SEEKTABLE 3 +#define DRFLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT 4 +#define DRFLAC_METADATA_BLOCK_TYPE_CUESHEET 5 +#define DRFLAC_METADATA_BLOCK_TYPE_PICTURE 6 +#define DRFLAC_METADATA_BLOCK_TYPE_INVALID 127 + +/* The various picture types specified in the PICTURE block. */ +#define DRFLAC_PICTURE_TYPE_OTHER 0 +#define DRFLAC_PICTURE_TYPE_FILE_ICON 1 +#define DRFLAC_PICTURE_TYPE_OTHER_FILE_ICON 2 +#define DRFLAC_PICTURE_TYPE_COVER_FRONT 3 +#define DRFLAC_PICTURE_TYPE_COVER_BACK 4 +#define DRFLAC_PICTURE_TYPE_LEAFLET_PAGE 5 +#define DRFLAC_PICTURE_TYPE_MEDIA 6 +#define DRFLAC_PICTURE_TYPE_LEAD_ARTIST 7 +#define DRFLAC_PICTURE_TYPE_ARTIST 8 +#define DRFLAC_PICTURE_TYPE_CONDUCTOR 9 +#define DRFLAC_PICTURE_TYPE_BAND 10 +#define DRFLAC_PICTURE_TYPE_COMPOSER 11 +#define DRFLAC_PICTURE_TYPE_LYRICIST 12 +#define DRFLAC_PICTURE_TYPE_RECORDING_LOCATION 13 +#define DRFLAC_PICTURE_TYPE_DURING_RECORDING 14 +#define DRFLAC_PICTURE_TYPE_DURING_PERFORMANCE 15 +#define DRFLAC_PICTURE_TYPE_SCREEN_CAPTURE 16 +#define DRFLAC_PICTURE_TYPE_BRIGHT_COLORED_FISH 17 +#define DRFLAC_PICTURE_TYPE_ILLUSTRATION 18 +#define DRFLAC_PICTURE_TYPE_BAND_LOGOTYPE 19 +#define DRFLAC_PICTURE_TYPE_PUBLISHER_LOGOTYPE 20 + +typedef enum +{ + drflac_container_native, + drflac_container_ogg, + drflac_container_unknown +} drflac_container; + +typedef enum +{ + drflac_seek_origin_start, + drflac_seek_origin_current +} drflac_seek_origin; + +/* The order of members in this structure is important because we map this directly to the raw data within the SEEKTABLE metadata block. */ +typedef struct +{ + drflac_uint64 firstPCMFrame; + drflac_uint64 flacFrameOffset; /* The offset from the first byte of the header of the first frame. */ + drflac_uint16 pcmFrameCount; +} drflac_seekpoint; + +typedef struct +{ + drflac_uint16 minBlockSizeInPCMFrames; + drflac_uint16 maxBlockSizeInPCMFrames; + drflac_uint32 minFrameSizeInPCMFrames; + drflac_uint32 maxFrameSizeInPCMFrames; + drflac_uint32 sampleRate; + drflac_uint8 channels; + drflac_uint8 bitsPerSample; + drflac_uint64 totalPCMFrameCount; + drflac_uint8 md5[16]; +} drflac_streaminfo; + +typedef struct +{ + /* + The metadata type. Use this to know how to interpret the data below. Will be set to one of the + DRFLAC_METADATA_BLOCK_TYPE_* tokens. + */ + drflac_uint32 type; + + /* + A pointer to the raw data. This points to a temporary buffer so don't hold on to it. It's best to + not modify the contents of this buffer. Use the structures below for more meaningful and structured + information about the metadata. It's possible for this to be null. + */ + const void* pRawData; + + /* The size in bytes of the block and the buffer pointed to by pRawData if it's non-NULL. */ + drflac_uint32 rawDataSize; + + union + { + drflac_streaminfo streaminfo; + + struct + { + int unused; + } padding; + + struct + { + drflac_uint32 id; + const void* pData; + drflac_uint32 dataSize; + } application; + + struct + { + drflac_uint32 seekpointCount; + const drflac_seekpoint* pSeekpoints; + } seektable; + + struct + { + drflac_uint32 vendorLength; + const char* vendor; + drflac_uint32 commentCount; + const void* pComments; + } vorbis_comment; + + struct + { + char catalog[128]; + drflac_uint64 leadInSampleCount; + drflac_bool32 isCD; + drflac_uint8 trackCount; + const void* pTrackData; + } cuesheet; + + struct + { + drflac_uint32 type; + drflac_uint32 mimeLength; + const char* mime; + drflac_uint32 descriptionLength; + const char* description; + drflac_uint32 width; + drflac_uint32 height; + drflac_uint32 colorDepth; + drflac_uint32 indexColorCount; + drflac_uint32 pictureDataSize; + const drflac_uint8* pPictureData; + } picture; + } data; +} drflac_metadata; + + +/* +Callback for when data needs to be read from the client. + + +Parameters +---------- +pUserData (in) + The user data that was passed to drflac_open() and family. + +pBufferOut (out) + The output buffer. + +bytesToRead (in) + The number of bytes to read. + + +Return Value +------------ +The number of bytes actually read. + + +Remarks +------- +A return value of less than bytesToRead indicates the end of the stream. Do _not_ return from this callback until either the entire bytesToRead is filled or +you have reached the end of the stream. +*/ +typedef size_t (* drflac_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead); + +/* +Callback for when data needs to be seeked. + + +Parameters +---------- +pUserData (in) + The user data that was passed to drflac_open() and family. + +offset (in) + The number of bytes to move, relative to the origin. Will never be negative. + +origin (in) + The origin of the seek - the current position or the start of the stream. + + +Return Value +------------ +Whether or not the seek was successful. + + +Remarks +------- +The offset will never be negative. Whether or not it is relative to the beginning or current position is determined by the "origin" parameter which will be +either drflac_seek_origin_start or drflac_seek_origin_current. + +When seeking to a PCM frame using drflac_seek_to_pcm_frame(), dr_flac may call this with an offset beyond the end of the FLAC stream. This needs to be detected +and handled by returning DRFLAC_FALSE. +*/ +typedef drflac_bool32 (* drflac_seek_proc)(void* pUserData, int offset, drflac_seek_origin origin); + +/* +Callback for when a metadata block is read. + + +Parameters +---------- +pUserData (in) + The user data that was passed to drflac_open() and family. + +pMetadata (in) + A pointer to a structure containing the data of the metadata block. + + +Remarks +------- +Use pMetadata->type to determine which metadata block is being handled and how to read the data. This +will be set to one of the DRFLAC_METADATA_BLOCK_TYPE_* tokens. +*/ +typedef void (* drflac_meta_proc)(void* pUserData, drflac_metadata* pMetadata); + + +/* Structure for internal use. Only used for decoders opened with drflac_open_memory. */ +typedef struct +{ + const drflac_uint8* data; + size_t dataSize; + size_t currentReadPos; +} drflac__memory_stream; + +/* Structure for internal use. Used for bit streaming. */ +typedef struct +{ + /* The function to call when more data needs to be read. */ + drflac_read_proc onRead; + + /* The function to call when the current read position needs to be moved. */ + drflac_seek_proc onSeek; + + /* The user data to pass around to onRead and onSeek. */ + void* pUserData; + + + /* + The number of unaligned bytes in the L2 cache. This will always be 0 until the end of the stream is hit. At the end of the + stream there will be a number of bytes that don't cleanly fit in an L1 cache line, so we use this variable to know whether + or not the bistreamer needs to run on a slower path to read those last bytes. This will never be more than sizeof(drflac_cache_t). + */ + size_t unalignedByteCount; + + /* The content of the unaligned bytes. */ + drflac_cache_t unalignedCache; + + /* The index of the next valid cache line in the "L2" cache. */ + drflac_uint32 nextL2Line; + + /* The number of bits that have been consumed by the cache. This is used to determine how many valid bits are remaining. */ + drflac_uint32 consumedBits; + + /* + The cached data which was most recently read from the client. There are two levels of cache. Data flows as such: + Client -> L2 -> L1. The L2 -> L1 movement is aligned and runs on a fast path in just a few instructions. + */ + drflac_cache_t cacheL2[DR_FLAC_BUFFER_SIZE/sizeof(drflac_cache_t)]; + drflac_cache_t cache; + + /* + CRC-16. This is updated whenever bits are read from the bit stream. Manually set this to 0 to reset the CRC. For FLAC, this + is reset to 0 at the beginning of each frame. + */ + drflac_uint16 crc16; + drflac_cache_t crc16Cache; /* A cache for optimizing CRC calculations. This is filled when when the L1 cache is reloaded. */ + drflac_uint32 crc16CacheIgnoredBytes; /* The number of bytes to ignore when updating the CRC-16 from the CRC-16 cache. */ +} drflac_bs; + +typedef struct +{ + /* The type of the subframe: SUBFRAME_CONSTANT, SUBFRAME_VERBATIM, SUBFRAME_FIXED or SUBFRAME_LPC. */ + drflac_uint8 subframeType; + + /* The number of wasted bits per sample as specified by the sub-frame header. */ + drflac_uint8 wastedBitsPerSample; + + /* The order to use for the prediction stage for SUBFRAME_FIXED and SUBFRAME_LPC. */ + drflac_uint8 lpcOrder; + + /* A pointer to the buffer containing the decoded samples in the subframe. This pointer is an offset from drflac::pExtraData. */ + drflac_int32* pSamplesS32; +} drflac_subframe; + +typedef struct +{ + /* + If the stream uses variable block sizes, this will be set to the index of the first PCM frame. If fixed block sizes are used, this will + always be set to 0. This is 64-bit because the decoded PCM frame number will be 36 bits. + */ + drflac_uint64 pcmFrameNumber; + + /* + If the stream uses fixed block sizes, this will be set to the frame number. If variable block sizes are used, this will always be 0. This + is 32-bit because in fixed block sizes, the maximum frame number will be 31 bits. + */ + drflac_uint32 flacFrameNumber; + + /* The sample rate of this frame. */ + drflac_uint32 sampleRate; + + /* The number of PCM frames in each sub-frame within this frame. */ + drflac_uint16 blockSizeInPCMFrames; + + /* + The channel assignment of this frame. This is not always set to the channel count. If interchannel decorrelation is being used this + will be set to DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE, DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE or DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE. + */ + drflac_uint8 channelAssignment; + + /* The number of bits per sample within this frame. */ + drflac_uint8 bitsPerSample; + + /* The frame's CRC. */ + drflac_uint8 crc8; +} drflac_frame_header; + +typedef struct +{ + /* The header. */ + drflac_frame_header header; + + /* + The number of PCM frames left to be read in this FLAC frame. This is initially set to the block size. As PCM frames are read, + this will be decremented. When it reaches 0, the decoder will see this frame as fully consumed and load the next frame. + */ + drflac_uint32 pcmFramesRemaining; + + /* The list of sub-frames within the frame. There is one sub-frame for each channel, and there's a maximum of 8 channels. */ + drflac_subframe subframes[8]; +} drflac_frame; + +typedef struct +{ + /* The function to call when a metadata block is read. */ + drflac_meta_proc onMeta; + + /* The user data posted to the metadata callback function. */ + void* pUserDataMD; + + /* Memory allocation callbacks. */ + drflac_allocation_callbacks allocationCallbacks; + + + /* The sample rate. Will be set to something like 44100. */ + drflac_uint32 sampleRate; + + /* + The number of channels. This will be set to 1 for monaural streams, 2 for stereo, etc. Maximum 8. This is set based on the + value specified in the STREAMINFO block. + */ + drflac_uint8 channels; + + /* The bits per sample. Will be set to something like 16, 24, etc. */ + drflac_uint8 bitsPerSample; + + /* The maximum block size, in samples. This number represents the number of samples in each channel (not combined). */ + drflac_uint16 maxBlockSizeInPCMFrames; + + /* + The total number of PCM Frames making up the stream. Can be 0 in which case it's still a valid stream, but just means + the total PCM frame count is unknown. Likely the case with streams like internet radio. + */ + drflac_uint64 totalPCMFrameCount; + + + /* The container type. This is set based on whether or not the decoder was opened from a native or Ogg stream. */ + drflac_container container; + + /* The number of seekpoints in the seektable. */ + drflac_uint32 seekpointCount; + + + /* Information about the frame the decoder is currently sitting on. */ + drflac_frame currentFLACFrame; + + + /* The index of the PCM frame the decoder is currently sitting on. This is only used for seeking. */ + drflac_uint64 currentPCMFrame; + + /* The position of the first FLAC frame in the stream. This is only ever used for seeking. */ + drflac_uint64 firstFLACFramePosInBytes; + + + /* A hack to avoid a malloc() when opening a decoder with drflac_open_memory(). */ + drflac__memory_stream memoryStream; + + + /* A pointer to the decoded sample data. This is an offset of pExtraData. */ + drflac_int32* pDecodedSamples; + + /* A pointer to the seek table. This is an offset of pExtraData, or NULL if there is no seek table. */ + drflac_seekpoint* pSeekpoints; + + /* Internal use only. Only used with Ogg containers. Points to a drflac_oggbs object. This is an offset of pExtraData. */ + void* _oggbs; + + /* Internal use only. Used for profiling and testing different seeking modes. */ + drflac_bool32 _noSeekTableSeek : 1; + drflac_bool32 _noBinarySearchSeek : 1; + drflac_bool32 _noBruteForceSeek : 1; + + /* The bit streamer. The raw FLAC data is fed through this object. */ + drflac_bs bs; + + /* Variable length extra data. We attach this to the end of the object so we can avoid unnecessary mallocs. */ + drflac_uint8 pExtraData[1]; +} drflac; + + +/* +Opens a FLAC decoder. + + +Parameters +---------- +onRead (in) + The function to call when data needs to be read from the client. + +onSeek (in) + The function to call when the read position of the client data needs to move. + +pUserData (in, optional) + A pointer to application defined data that will be passed to onRead and onSeek. + +pAllocationCallbacks (in, optional) + A pointer to application defined callbacks for managing memory allocations. + + +Return Value +------------ +Returns a pointer to an object representing the decoder. + + +Remarks +------- +Close the decoder with `drflac_close()`. + +`pAllocationCallbacks` can be NULL in which case it will use `DRFLAC_MALLOC`, `DRFLAC_REALLOC` and `DRFLAC_FREE`. + +This function will automatically detect whether or not you are attempting to open a native or Ogg encapsulated FLAC, both of which should work seamlessly +without any manual intervention. Ogg encapsulation also works with multiplexed streams which basically means it can play FLAC encoded audio tracks in videos. + +This is the lowest level function for opening a FLAC stream. You can also use `drflac_open_file()` and `drflac_open_memory()` to open the stream from a file or +from a block of memory respectively. + +The STREAMINFO block must be present for this to succeed. Use `drflac_open_relaxed()` to open a FLAC stream where the header may not be present. + +Use `drflac_open_with_metadata()` if you need access to metadata. + + +Seek Also +--------- +drflac_open_file() +drflac_open_memory() +drflac_open_with_metadata() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Opens a FLAC stream with relaxed validation of the header block. + + +Parameters +---------- +onRead (in) + The function to call when data needs to be read from the client. + +onSeek (in) + The function to call when the read position of the client data needs to move. + +container (in) + Whether or not the FLAC stream is encapsulated using standard FLAC encapsulation or Ogg encapsulation. + +pUserData (in, optional) + A pointer to application defined data that will be passed to onRead and onSeek. + +pAllocationCallbacks (in, optional) + A pointer to application defined callbacks for managing memory allocations. + + +Return Value +------------ +A pointer to an object representing the decoder. + + +Remarks +------- +The same as drflac_open(), except attempts to open the stream even when a header block is not present. + +Because the header is not necessarily available, the caller must explicitly define the container (Native or Ogg). Do not set this to `drflac_container_unknown` +as that is for internal use only. + +Opening in relaxed mode will continue reading data from onRead until it finds a valid frame. If a frame is never found it will continue forever. To abort, +force your `onRead` callback to return 0, which dr_flac will use as an indicator that the end of the stream was found. + +Use `drflac_open_with_metadata_relaxed()` if you need access to metadata. +*/ +DRFLAC_API drflac* drflac_open_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_container container, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Opens a FLAC decoder and notifies the caller of the metadata chunks (album art, etc.). + + +Parameters +---------- +onRead (in) + The function to call when data needs to be read from the client. + +onSeek (in) + The function to call when the read position of the client data needs to move. + +onMeta (in) + The function to call for every metadata block. + +pUserData (in, optional) + A pointer to application defined data that will be passed to onRead, onSeek and onMeta. + +pAllocationCallbacks (in, optional) + A pointer to application defined callbacks for managing memory allocations. + + +Return Value +------------ +A pointer to an object representing the decoder. + + +Remarks +------- +Close the decoder with `drflac_close()`. + +`pAllocationCallbacks` can be NULL in which case it will use `DRFLAC_MALLOC`, `DRFLAC_REALLOC` and `DRFLAC_FREE`. + +This is slower than `drflac_open()`, so avoid this one if you don't need metadata. Internally, this will allocate and free memory on the heap for every +metadata block except for STREAMINFO and PADDING blocks. + +The caller is notified of the metadata via the `onMeta` callback. All metadata blocks will be handled before the function returns. This callback takes a +pointer to a `drflac_metadata` object which is a union containing the data of all relevant metadata blocks. Use the `type` member to discriminate against +the different metadata types. + +The STREAMINFO block must be present for this to succeed. Use `drflac_open_with_metadata_relaxed()` to open a FLAC stream where the header may not be present. + +Note that this will behave inconsistently with `drflac_open()` if the stream is an Ogg encapsulated stream and a metadata block is corrupted. This is due to +the way the Ogg stream recovers from corrupted pages. When `drflac_open_with_metadata()` is being used, the open routine will try to read the contents of the +metadata block, whereas `drflac_open()` will simply seek past it (for the sake of efficiency). This inconsistency can result in different samples being +returned depending on whether or not the stream is being opened with metadata. + + +Seek Also +--------- +drflac_open_file_with_metadata() +drflac_open_memory_with_metadata() +drflac_open() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open_with_metadata(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +The same as drflac_open_with_metadata(), except attempts to open the stream even when a header block is not present. + +See Also +-------- +drflac_open_with_metadata() +drflac_open_relaxed() +*/ +DRFLAC_API drflac* drflac_open_with_metadata_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Closes the given FLAC decoder. + + +Parameters +---------- +pFlac (in) + The decoder to close. + + +Remarks +------- +This will destroy the decoder object. + + +See Also +-------- +drflac_open() +drflac_open_with_metadata() +drflac_open_file() +drflac_open_file_w() +drflac_open_file_with_metadata() +drflac_open_file_with_metadata_w() +drflac_open_memory() +drflac_open_memory_with_metadata() +*/ +DRFLAC_API void drflac_close(drflac* pFlac); + + +/* +Reads sample data from the given FLAC decoder, output as interleaved signed 32-bit PCM. + + +Parameters +---------- +pFlac (in) + The decoder. + +framesToRead (in) + The number of PCM frames to read. + +pBufferOut (out, optional) + A pointer to the buffer that will receive the decoded samples. + + +Return Value +------------ +Returns the number of PCM frames actually read. If the return value is less than `framesToRead` it has reached the end. + + +Remarks +------- +pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of frames seeked. +*/ +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_s32(drflac* pFlac, drflac_uint64 framesToRead, drflac_int32* pBufferOut); + + +/* +Reads sample data from the given FLAC decoder, output as interleaved signed 16-bit PCM. + + +Parameters +---------- +pFlac (in) + The decoder. + +framesToRead (in) + The number of PCM frames to read. + +pBufferOut (out, optional) + A pointer to the buffer that will receive the decoded samples. + + +Return Value +------------ +Returns the number of PCM frames actually read. If the return value is less than `framesToRead` it has reached the end. + + +Remarks +------- +pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of frames seeked. + +Note that this is lossy for streams where the bits per sample is larger than 16. +*/ +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_s16(drflac* pFlac, drflac_uint64 framesToRead, drflac_int16* pBufferOut); + +/* +Reads sample data from the given FLAC decoder, output as interleaved 32-bit floating point PCM. + + +Parameters +---------- +pFlac (in) + The decoder. + +framesToRead (in) + The number of PCM frames to read. + +pBufferOut (out, optional) + A pointer to the buffer that will receive the decoded samples. + + +Return Value +------------ +Returns the number of PCM frames actually read. If the return value is less than `framesToRead` it has reached the end. + + +Remarks +------- +pBufferOut can be null, in which case the call will act as a seek, and the return value will be the number of frames seeked. + +Note that this should be considered lossy due to the nature of floating point numbers not being able to exactly represent every possible number. +*/ +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_f32(drflac* pFlac, drflac_uint64 framesToRead, float* pBufferOut); + +/* +Seeks to the PCM frame at the given index. + + +Parameters +---------- +pFlac (in) + The decoder. + +pcmFrameIndex (in) + The index of the PCM frame to seek to. See notes below. + + +Return Value +------------- +`DRFLAC_TRUE` if successful; `DRFLAC_FALSE` otherwise. +*/ +DRFLAC_API drflac_bool32 drflac_seek_to_pcm_frame(drflac* pFlac, drflac_uint64 pcmFrameIndex); + + + +#ifndef DR_FLAC_NO_STDIO +/* +Opens a FLAC decoder from the file at the given path. + + +Parameters +---------- +pFileName (in) + The path of the file to open, either absolute or relative to the current directory. + +pAllocationCallbacks (in, optional) + A pointer to application defined callbacks for managing memory allocations. + + +Return Value +------------ +A pointer to an object representing the decoder. + + +Remarks +------- +Close the decoder with drflac_close(). + + +Remarks +------- +This will hold a handle to the file until the decoder is closed with drflac_close(). Some platforms will restrict the number of files a process can have open +at any given time, so keep this mind if you have many decoders open at the same time. + + +See Also +-------- +drflac_open_file_with_metadata() +drflac_open() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open_file(const char* pFileName, const drflac_allocation_callbacks* pAllocationCallbacks); +DRFLAC_API drflac* drflac_open_file_w(const wchar_t* pFileName, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Opens a FLAC decoder from the file at the given path and notifies the caller of the metadata chunks (album art, etc.) + + +Parameters +---------- +pFileName (in) + The path of the file to open, either absolute or relative to the current directory. + +pAllocationCallbacks (in, optional) + A pointer to application defined callbacks for managing memory allocations. + +onMeta (in) + The callback to fire for each metadata block. + +pUserData (in) + A pointer to the user data to pass to the metadata callback. + +pAllocationCallbacks (in) + A pointer to application defined callbacks for managing memory allocations. + + +Remarks +------- +Look at the documentation for drflac_open_with_metadata() for more information on how metadata is handled. + + +See Also +-------- +drflac_open_with_metadata() +drflac_open() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open_file_with_metadata(const char* pFileName, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); +DRFLAC_API drflac* drflac_open_file_with_metadata_w(const wchar_t* pFileName, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); +#endif + +/* +Opens a FLAC decoder from a pre-allocated block of memory + + +Parameters +---------- +pData (in) + A pointer to the raw encoded FLAC data. + +dataSize (in) + The size in bytes of `data`. + +pAllocationCallbacks (in) + A pointer to application defined callbacks for managing memory allocations. + + +Return Value +------------ +A pointer to an object representing the decoder. + + +Remarks +------- +This does not create a copy of the data. It is up to the application to ensure the buffer remains valid for the lifetime of the decoder. + + +See Also +-------- +drflac_open() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open_memory(const void* pData, size_t dataSize, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Opens a FLAC decoder from a pre-allocated block of memory and notifies the caller of the metadata chunks (album art, etc.) + + +Parameters +---------- +pData (in) + A pointer to the raw encoded FLAC data. + +dataSize (in) + The size in bytes of `data`. + +onMeta (in) + The callback to fire for each metadata block. + +pUserData (in) + A pointer to the user data to pass to the metadata callback. + +pAllocationCallbacks (in) + A pointer to application defined callbacks for managing memory allocations. + + +Remarks +------- +Look at the documentation for drflac_open_with_metadata() for more information on how metadata is handled. + + +See Also +------- +drflac_open_with_metadata() +drflac_open() +drflac_close() +*/ +DRFLAC_API drflac* drflac_open_memory_with_metadata(const void* pData, size_t dataSize, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks); + + + +/* High Level APIs */ + +/* +Opens a FLAC stream from the given callbacks and fully decodes it in a single operation. The return value is a +pointer to the sample data as interleaved signed 32-bit PCM. The returned data must be freed with drflac_free(). + +You can pass in custom memory allocation callbacks via the pAllocationCallbacks parameter. This can be NULL in which +case it will use DRFLAC_MALLOC, DRFLAC_REALLOC and DRFLAC_FREE. + +Sometimes a FLAC file won't keep track of the total sample count. In this situation the function will continuously +read samples into a dynamically sized buffer on the heap until no samples are left. + +Do not call this function on a broadcast type of stream (like internet radio streams and whatnot). +*/ +DRFLAC_API drflac_int32* drflac_open_and_read_pcm_frames_s32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_and_read_pcm_frames_s32(), except returns signed 16-bit integer samples. */ +DRFLAC_API drflac_int16* drflac_open_and_read_pcm_frames_s16(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_and_read_pcm_frames_s32(), except returns 32-bit floating-point samples. */ +DRFLAC_API float* drflac_open_and_read_pcm_frames_f32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +#ifndef DR_FLAC_NO_STDIO +/* Same as drflac_open_and_read_pcm_frames_s32() except opens the decoder from a file. */ +DRFLAC_API drflac_int32* drflac_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_file_and_read_pcm_frames_s32(), except returns signed 16-bit integer samples. */ +DRFLAC_API drflac_int16* drflac_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_file_and_read_pcm_frames_s32(), except returns 32-bit floating-point samples. */ +DRFLAC_API float* drflac_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); +#endif + +/* Same as drflac_open_and_read_pcm_frames_s32() except opens the decoder from a block of memory. */ +DRFLAC_API drflac_int32* drflac_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_memory_and_read_pcm_frames_s32(), except returns signed 16-bit integer samples. */ +DRFLAC_API drflac_int16* drflac_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* Same as drflac_open_memory_and_read_pcm_frames_s32(), except returns 32-bit floating-point samples. */ +DRFLAC_API float* drflac_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks); + +/* +Frees memory that was allocated internally by dr_flac. + +Set pAllocationCallbacks to the same object that was passed to drflac_open_*_and_read_pcm_frames_*(). If you originally passed in NULL, pass in NULL for this. +*/ +DRFLAC_API void drflac_free(void* p, const drflac_allocation_callbacks* pAllocationCallbacks); + + +/* Structure representing an iterator for vorbis comments in a VORBIS_COMMENT metadata block. */ +typedef struct +{ + drflac_uint32 countRemaining; + const char* pRunningData; +} drflac_vorbis_comment_iterator; + +/* +Initializes a vorbis comment iterator. This can be used for iterating over the vorbis comments in a VORBIS_COMMENT +metadata block. +*/ +DRFLAC_API void drflac_init_vorbis_comment_iterator(drflac_vorbis_comment_iterator* pIter, drflac_uint32 commentCount, const void* pComments); + +/* +Goes to the next vorbis comment in the given iterator. If null is returned it means there are no more comments. The +returned string is NOT null terminated. +*/ +DRFLAC_API const char* drflac_next_vorbis_comment(drflac_vorbis_comment_iterator* pIter, drflac_uint32* pCommentLengthOut); + + +/* Structure representing an iterator for cuesheet tracks in a CUESHEET metadata block. */ +typedef struct +{ + drflac_uint32 countRemaining; + const char* pRunningData; +} drflac_cuesheet_track_iterator; + +/* The order of members here is important because we map this directly to the raw data within the CUESHEET metadata block. */ +typedef struct +{ + drflac_uint64 offset; + drflac_uint8 index; + drflac_uint8 reserved[3]; +} drflac_cuesheet_track_index; + +typedef struct +{ + drflac_uint64 offset; + drflac_uint8 trackNumber; + char ISRC[12]; + drflac_bool8 isAudio; + drflac_bool8 preEmphasis; + drflac_uint8 indexCount; + const drflac_cuesheet_track_index* pIndexPoints; +} drflac_cuesheet_track; + +/* +Initializes a cuesheet track iterator. This can be used for iterating over the cuesheet tracks in a CUESHEET metadata +block. +*/ +DRFLAC_API void drflac_init_cuesheet_track_iterator(drflac_cuesheet_track_iterator* pIter, drflac_uint32 trackCount, const void* pTrackData); + +/* Goes to the next cuesheet track in the given iterator. If DRFLAC_FALSE is returned it means there are no more comments. */ +DRFLAC_API drflac_bool32 drflac_next_cuesheet_track(drflac_cuesheet_track_iterator* pIter, drflac_cuesheet_track* pCuesheetTrack); + + +#ifdef __cplusplus +} +#endif +#endif /* dr_flac_h */ + + +/************************************************************************************************************************************************************ + ************************************************************************************************************************************************************ + + IMPLEMENTATION + + ************************************************************************************************************************************************************ + ************************************************************************************************************************************************************/ +#if defined(DR_FLAC_IMPLEMENTATION) || defined(DRFLAC_IMPLEMENTATION) +#ifndef dr_flac_c +#define dr_flac_c + +/* Disable some annoying warnings. */ +#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))) + #pragma GCC diagnostic push + #if __GNUC__ >= 7 + #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" + #endif +#endif + +#ifdef __linux__ + #ifndef _BSD_SOURCE + #define _BSD_SOURCE + #endif + #ifndef _DEFAULT_SOURCE + #define _DEFAULT_SOURCE + #endif + #ifndef __USE_BSD + #define __USE_BSD + #endif + #include +#endif + +#include +#include + +/* Inline */ +#ifdef _MSC_VER + #define DRFLAC_INLINE __forceinline +#elif defined(__GNUC__) + /* + I've had a bug report where GCC is emitting warnings about functions possibly not being inlineable. This warning happens when + the __attribute__((always_inline)) attribute is defined without an "inline" statement. I think therefore there must be some + case where "__inline__" is not always defined, thus the compiler emitting these warnings. When using -std=c89 or -ansi on the + command line, we cannot use the "inline" keyword and instead need to use "__inline__". In an attempt to work around this issue + I am using "__inline__" only when we're compiling in strict ANSI mode. + */ + #if defined(__STRICT_ANSI__) + #define DRFLAC_GNUC_INLINE_HINT __inline__ + #else + #define DRFLAC_GNUC_INLINE_HINT inline + #endif + + #if (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 2)) || defined(__clang__) + #define DRFLAC_INLINE DRFLAC_GNUC_INLINE_HINT __attribute__((always_inline)) + #else + #define DRFLAC_INLINE DRFLAC_GNUC_INLINE_HINT + #endif +#elif defined(__WATCOMC__) + #define DRFLAC_INLINE __inline +#else + #define DRFLAC_INLINE +#endif +/* End Inline */ + +/* +Intrinsics Support + +There's a bug in GCC 4.2.x which results in an incorrect compilation error when using _mm_slli_epi32() where it complains with + + "error: shift must be an immediate" + +Unfortuantely dr_flac depends on this for a few things so we're just going to disable SSE on GCC 4.2 and below. +*/ +#if !defined(DR_FLAC_NO_SIMD) + #if defined(DRFLAC_X64) || defined(DRFLAC_X86) + #if defined(_MSC_VER) && !defined(__clang__) + /* MSVC. */ + #if _MSC_VER >= 1400 && !defined(DRFLAC_NO_SSE2) /* 2005 */ + #define DRFLAC_SUPPORT_SSE2 + #endif + #if _MSC_VER >= 1600 && !defined(DRFLAC_NO_SSE41) /* 2010 */ + #define DRFLAC_SUPPORT_SSE41 + #endif + #elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) + /* Assume GNUC-style. */ + #if defined(__SSE2__) && !defined(DRFLAC_NO_SSE2) + #define DRFLAC_SUPPORT_SSE2 + #endif + #if defined(__SSE4_1__) && !defined(DRFLAC_NO_SSE41) + #define DRFLAC_SUPPORT_SSE41 + #endif + #endif + + /* If at this point we still haven't determined compiler support for the intrinsics just fall back to __has_include. */ + #if !defined(__GNUC__) && !defined(__clang__) && defined(__has_include) + #if !defined(DRFLAC_SUPPORT_SSE2) && !defined(DRFLAC_NO_SSE2) && __has_include() + #define DRFLAC_SUPPORT_SSE2 + #endif + #if !defined(DRFLAC_SUPPORT_SSE41) && !defined(DRFLAC_NO_SSE41) && __has_include() + #define DRFLAC_SUPPORT_SSE41 + #endif + #endif + + #if defined(DRFLAC_SUPPORT_SSE41) + #include + #elif defined(DRFLAC_SUPPORT_SSE2) + #include + #endif + #endif + + #if defined(DRFLAC_ARM) + #if !defined(DRFLAC_NO_NEON) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64)) + #define DRFLAC_SUPPORT_NEON + #include + #endif + #endif +#endif + +/* Compile-time CPU feature support. */ +#if !defined(DR_FLAC_NO_SIMD) && (defined(DRFLAC_X86) || defined(DRFLAC_X64)) + #if defined(_MSC_VER) && !defined(__clang__) + #if _MSC_VER >= 1400 + #include + static void drflac__cpuid(int info[4], int fid) + { + __cpuid(info, fid); + } + #else + #define DRFLAC_NO_CPUID + #endif + #else + #if defined(__GNUC__) || defined(__clang__) + static void drflac__cpuid(int info[4], int fid) + { + /* + It looks like the -fPIC option uses the ebx register which GCC complains about. We can work around this by just using a different register, the + specific register of which I'm letting the compiler decide on. The "k" prefix is used to specify a 32-bit register. The {...} syntax is for + supporting different assembly dialects. + + What's basically happening is that we're saving and restoring the ebx register manually. + */ + #if defined(DRFLAC_X86) && defined(__PIC__) + __asm__ __volatile__ ( + "xchg{l} {%%}ebx, %k1;" + "cpuid;" + "xchg{l} {%%}ebx, %k1;" + : "=a"(info[0]), "=&r"(info[1]), "=c"(info[2]), "=d"(info[3]) : "a"(fid), "c"(0) + ); + #else + __asm__ __volatile__ ( + "cpuid" : "=a"(info[0]), "=b"(info[1]), "=c"(info[2]), "=d"(info[3]) : "a"(fid), "c"(0) + ); + #endif + } + #else + #define DRFLAC_NO_CPUID + #endif + #endif +#else + #define DRFLAC_NO_CPUID +#endif + +static DRFLAC_INLINE drflac_bool32 drflac_has_sse2(void) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + #if (defined(DRFLAC_X64) || defined(DRFLAC_X86)) && !defined(DRFLAC_NO_SSE2) + #if defined(DRFLAC_X64) + return DRFLAC_TRUE; /* 64-bit targets always support SSE2. */ + #elif (defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__) + return DRFLAC_TRUE; /* If the compiler is allowed to freely generate SSE2 code we can assume support. */ + #else + #if defined(DRFLAC_NO_CPUID) + return DRFLAC_FALSE; + #else + int info[4]; + drflac__cpuid(info, 1); + return (info[3] & (1 << 26)) != 0; + #endif + #endif + #else + return DRFLAC_FALSE; /* SSE2 is only supported on x86 and x64 architectures. */ + #endif +#else + return DRFLAC_FALSE; /* No compiler support. */ +#endif +} + +static DRFLAC_INLINE drflac_bool32 drflac_has_sse41(void) +{ +#if defined(DRFLAC_SUPPORT_SSE41) + #if (defined(DRFLAC_X64) || defined(DRFLAC_X86)) && !defined(DRFLAC_NO_SSE41) + #if defined(__SSE4_1__) || defined(__AVX__) + return DRFLAC_TRUE; /* If the compiler is allowed to freely generate SSE41 code we can assume support. */ + #else + #if defined(DRFLAC_NO_CPUID) + return DRFLAC_FALSE; + #else + int info[4]; + drflac__cpuid(info, 1); + return (info[2] & (1 << 19)) != 0; + #endif + #endif + #else + return DRFLAC_FALSE; /* SSE41 is only supported on x86 and x64 architectures. */ + #endif +#else + return DRFLAC_FALSE; /* No compiler support. */ +#endif +} + + +#if defined(_MSC_VER) && _MSC_VER >= 1500 && (defined(DRFLAC_X86) || defined(DRFLAC_X64)) && !defined(__clang__) + #define DRFLAC_HAS_LZCNT_INTRINSIC +#elif (defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))) + #define DRFLAC_HAS_LZCNT_INTRINSIC +#elif defined(__clang__) + #if defined(__has_builtin) + #if __has_builtin(__builtin_clzll) || __has_builtin(__builtin_clzl) + #define DRFLAC_HAS_LZCNT_INTRINSIC + #endif + #endif +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1400 && !defined(__clang__) + #define DRFLAC_HAS_BYTESWAP16_INTRINSIC + #define DRFLAC_HAS_BYTESWAP32_INTRINSIC + #define DRFLAC_HAS_BYTESWAP64_INTRINSIC +#elif defined(__clang__) + #if defined(__has_builtin) + #if __has_builtin(__builtin_bswap16) + #define DRFLAC_HAS_BYTESWAP16_INTRINSIC + #endif + #if __has_builtin(__builtin_bswap32) + #define DRFLAC_HAS_BYTESWAP32_INTRINSIC + #endif + #if __has_builtin(__builtin_bswap64) + #define DRFLAC_HAS_BYTESWAP64_INTRINSIC + #endif + #endif +#elif defined(__GNUC__) + #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define DRFLAC_HAS_BYTESWAP32_INTRINSIC + #define DRFLAC_HAS_BYTESWAP64_INTRINSIC + #endif + #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) + #define DRFLAC_HAS_BYTESWAP16_INTRINSIC + #endif +#elif defined(__WATCOMC__) && defined(__386__) + #define DRFLAC_HAS_BYTESWAP16_INTRINSIC + #define DRFLAC_HAS_BYTESWAP32_INTRINSIC + #define DRFLAC_HAS_BYTESWAP64_INTRINSIC + extern __inline drflac_uint16 _watcom_bswap16(drflac_uint16); + extern __inline drflac_uint32 _watcom_bswap32(drflac_uint32); + extern __inline drflac_uint64 _watcom_bswap64(drflac_uint64); +#pragma aux _watcom_bswap16 = \ + "xchg al, ah" \ + parm [ax] \ + value [ax] \ + modify nomemory; +#pragma aux _watcom_bswap32 = \ + "bswap eax" \ + parm [eax] \ + value [eax] \ + modify nomemory; +#pragma aux _watcom_bswap64 = \ + "bswap eax" \ + "bswap edx" \ + "xchg eax,edx" \ + parm [eax edx] \ + value [eax edx] \ + modify nomemory; +#endif + + +/* Standard library stuff. */ +#ifndef DRFLAC_ASSERT +#include +#define DRFLAC_ASSERT(expression) assert(expression) +#endif +#ifndef DRFLAC_MALLOC +#define DRFLAC_MALLOC(sz) malloc((sz)) +#endif +#ifndef DRFLAC_REALLOC +#define DRFLAC_REALLOC(p, sz) realloc((p), (sz)) +#endif +#ifndef DRFLAC_FREE +#define DRFLAC_FREE(p) free((p)) +#endif +#ifndef DRFLAC_COPY_MEMORY +#define DRFLAC_COPY_MEMORY(dst, src, sz) memcpy((dst), (src), (sz)) +#endif +#ifndef DRFLAC_ZERO_MEMORY +#define DRFLAC_ZERO_MEMORY(p, sz) memset((p), 0, (sz)) +#endif +#ifndef DRFLAC_ZERO_OBJECT +#define DRFLAC_ZERO_OBJECT(p) DRFLAC_ZERO_MEMORY((p), sizeof(*(p))) +#endif + +#define DRFLAC_MAX_SIMD_VECTOR_SIZE 64 /* 64 for AVX-512 in the future. */ + +/* Result Codes */ +typedef drflac_int32 drflac_result; +#define DRFLAC_SUCCESS 0 +#define DRFLAC_ERROR -1 /* A generic error. */ +#define DRFLAC_INVALID_ARGS -2 +#define DRFLAC_INVALID_OPERATION -3 +#define DRFLAC_OUT_OF_MEMORY -4 +#define DRFLAC_OUT_OF_RANGE -5 +#define DRFLAC_ACCESS_DENIED -6 +#define DRFLAC_DOES_NOT_EXIST -7 +#define DRFLAC_ALREADY_EXISTS -8 +#define DRFLAC_TOO_MANY_OPEN_FILES -9 +#define DRFLAC_INVALID_FILE -10 +#define DRFLAC_TOO_BIG -11 +#define DRFLAC_PATH_TOO_LONG -12 +#define DRFLAC_NAME_TOO_LONG -13 +#define DRFLAC_NOT_DIRECTORY -14 +#define DRFLAC_IS_DIRECTORY -15 +#define DRFLAC_DIRECTORY_NOT_EMPTY -16 +#define DRFLAC_END_OF_FILE -17 +#define DRFLAC_NO_SPACE -18 +#define DRFLAC_BUSY -19 +#define DRFLAC_IO_ERROR -20 +#define DRFLAC_INTERRUPT -21 +#define DRFLAC_UNAVAILABLE -22 +#define DRFLAC_ALREADY_IN_USE -23 +#define DRFLAC_BAD_ADDRESS -24 +#define DRFLAC_BAD_SEEK -25 +#define DRFLAC_BAD_PIPE -26 +#define DRFLAC_DEADLOCK -27 +#define DRFLAC_TOO_MANY_LINKS -28 +#define DRFLAC_NOT_IMPLEMENTED -29 +#define DRFLAC_NO_MESSAGE -30 +#define DRFLAC_BAD_MESSAGE -31 +#define DRFLAC_NO_DATA_AVAILABLE -32 +#define DRFLAC_INVALID_DATA -33 +#define DRFLAC_TIMEOUT -34 +#define DRFLAC_NO_NETWORK -35 +#define DRFLAC_NOT_UNIQUE -36 +#define DRFLAC_NOT_SOCKET -37 +#define DRFLAC_NO_ADDRESS -38 +#define DRFLAC_BAD_PROTOCOL -39 +#define DRFLAC_PROTOCOL_UNAVAILABLE -40 +#define DRFLAC_PROTOCOL_NOT_SUPPORTED -41 +#define DRFLAC_PROTOCOL_FAMILY_NOT_SUPPORTED -42 +#define DRFLAC_ADDRESS_FAMILY_NOT_SUPPORTED -43 +#define DRFLAC_SOCKET_NOT_SUPPORTED -44 +#define DRFLAC_CONNECTION_RESET -45 +#define DRFLAC_ALREADY_CONNECTED -46 +#define DRFLAC_NOT_CONNECTED -47 +#define DRFLAC_CONNECTION_REFUSED -48 +#define DRFLAC_NO_HOST -49 +#define DRFLAC_IN_PROGRESS -50 +#define DRFLAC_CANCELLED -51 +#define DRFLAC_MEMORY_ALREADY_MAPPED -52 +#define DRFLAC_AT_END -53 + +#define DRFLAC_CRC_MISMATCH -100 +/* End Result Codes */ + + +#define DRFLAC_SUBFRAME_CONSTANT 0 +#define DRFLAC_SUBFRAME_VERBATIM 1 +#define DRFLAC_SUBFRAME_FIXED 8 +#define DRFLAC_SUBFRAME_LPC 32 +#define DRFLAC_SUBFRAME_RESERVED 255 + +#define DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE 0 +#define DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2 1 + +#define DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT 0 +#define DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE 8 +#define DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE 9 +#define DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE 10 + +#define DRFLAC_SEEKPOINT_SIZE_IN_BYTES 18 +#define DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES 36 +#define DRFLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES 12 + +#define drflac_align(x, a) ((((x) + (a) - 1) / (a)) * (a)) + + +DRFLAC_API void drflac_version(drflac_uint32* pMajor, drflac_uint32* pMinor, drflac_uint32* pRevision) +{ + if (pMajor) { + *pMajor = DRFLAC_VERSION_MAJOR; + } + + if (pMinor) { + *pMinor = DRFLAC_VERSION_MINOR; + } + + if (pRevision) { + *pRevision = DRFLAC_VERSION_REVISION; + } +} + +DRFLAC_API const char* drflac_version_string(void) +{ + return DRFLAC_VERSION_STRING; +} + + +/* CPU caps. */ +#if defined(__has_feature) + #if __has_feature(thread_sanitizer) + #define DRFLAC_NO_THREAD_SANITIZE __attribute__((no_sanitize("thread"))) + #else + #define DRFLAC_NO_THREAD_SANITIZE + #endif +#else + #define DRFLAC_NO_THREAD_SANITIZE +#endif + +#if defined(DRFLAC_HAS_LZCNT_INTRINSIC) +static drflac_bool32 drflac__gIsLZCNTSupported = DRFLAC_FALSE; +#endif + +#ifndef DRFLAC_NO_CPUID +static drflac_bool32 drflac__gIsSSE2Supported = DRFLAC_FALSE; +static drflac_bool32 drflac__gIsSSE41Supported = DRFLAC_FALSE; + +/* +I've had a bug report that Clang's ThreadSanitizer presents a warning in this function. Having reviewed this, this does +actually make sense. However, since CPU caps should never differ for a running process, I don't think the trade off of +complicating internal API's by passing around CPU caps versus just disabling the warnings is worthwhile. I'm therefore +just going to disable these warnings. This is disabled via the DRFLAC_NO_THREAD_SANITIZE attribute. +*/ +DRFLAC_NO_THREAD_SANITIZE static void drflac__init_cpu_caps(void) +{ + static drflac_bool32 isCPUCapsInitialized = DRFLAC_FALSE; + + if (!isCPUCapsInitialized) { + /* LZCNT */ +#if defined(DRFLAC_HAS_LZCNT_INTRINSIC) + int info[4] = {0}; + drflac__cpuid(info, 0x80000001); + drflac__gIsLZCNTSupported = (info[2] & (1 << 5)) != 0; +#endif + + /* SSE2 */ + drflac__gIsSSE2Supported = drflac_has_sse2(); + + /* SSE4.1 */ + drflac__gIsSSE41Supported = drflac_has_sse41(); + + /* Initialized. */ + isCPUCapsInitialized = DRFLAC_TRUE; + } +} +#else +static drflac_bool32 drflac__gIsNEONSupported = DRFLAC_FALSE; + +static DRFLAC_INLINE drflac_bool32 drflac__has_neon(void) +{ +#if defined(DRFLAC_SUPPORT_NEON) + #if defined(DRFLAC_ARM) && !defined(DRFLAC_NO_NEON) + #if (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64)) + return DRFLAC_TRUE; /* If the compiler is allowed to freely generate NEON code we can assume support. */ + #else + /* TODO: Runtime check. */ + return DRFLAC_FALSE; + #endif + #else + return DRFLAC_FALSE; /* NEON is only supported on ARM architectures. */ + #endif +#else + return DRFLAC_FALSE; /* No compiler support. */ +#endif +} + +DRFLAC_NO_THREAD_SANITIZE static void drflac__init_cpu_caps(void) +{ + drflac__gIsNEONSupported = drflac__has_neon(); + +#if defined(DRFLAC_HAS_LZCNT_INTRINSIC) && defined(DRFLAC_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5) + drflac__gIsLZCNTSupported = DRFLAC_TRUE; +#endif +} +#endif + + +/* Endian Management */ +static DRFLAC_INLINE drflac_bool32 drflac__is_little_endian(void) +{ +#if defined(DRFLAC_X86) || defined(DRFLAC_X64) + return DRFLAC_TRUE; +#elif defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN + return DRFLAC_TRUE; +#else + int n = 1; + return (*(char*)&n) == 1; +#endif +} + +static DRFLAC_INLINE drflac_uint16 drflac__swap_endian_uint16(drflac_uint16 n) +{ +#ifdef DRFLAC_HAS_BYTESWAP16_INTRINSIC + #if defined(_MSC_VER) && !defined(__clang__) + return _byteswap_ushort(n); + #elif defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(n); + #elif defined(__WATCOMC__) && defined(__386__) + return _watcom_bswap16(n); + #else + #error "This compiler does not support the byte swap intrinsic." + #endif +#else + return ((n & 0xFF00) >> 8) | + ((n & 0x00FF) << 8); +#endif +} + +static DRFLAC_INLINE drflac_uint32 drflac__swap_endian_uint32(drflac_uint32 n) +{ +#ifdef DRFLAC_HAS_BYTESWAP32_INTRINSIC + #if defined(_MSC_VER) && !defined(__clang__) + return _byteswap_ulong(n); + #elif defined(__GNUC__) || defined(__clang__) + #if defined(DRFLAC_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 6) && !defined(__ARM_ARCH_6M__) && !defined(DRFLAC_64BIT) /* <-- 64-bit inline assembly has not been tested, so disabling for now. */ + /* Inline assembly optimized implementation for ARM. In my testing, GCC does not generate optimized code with __builtin_bswap32(). */ + drflac_uint32 r; + __asm__ __volatile__ ( + #if defined(DRFLAC_64BIT) + "rev %w[out], %w[in]" : [out]"=r"(r) : [in]"r"(n) /* <-- This is untested. If someone in the community could test this, that would be appreciated! */ + #else + "rev %[out], %[in]" : [out]"=r"(r) : [in]"r"(n) + #endif + ); + return r; + #else + return __builtin_bswap32(n); + #endif + #elif defined(__WATCOMC__) && defined(__386__) + return _watcom_bswap32(n); + #else + #error "This compiler does not support the byte swap intrinsic." + #endif +#else + return ((n & 0xFF000000) >> 24) | + ((n & 0x00FF0000) >> 8) | + ((n & 0x0000FF00) << 8) | + ((n & 0x000000FF) << 24); +#endif +} + +static DRFLAC_INLINE drflac_uint64 drflac__swap_endian_uint64(drflac_uint64 n) +{ +#ifdef DRFLAC_HAS_BYTESWAP64_INTRINSIC + #if defined(_MSC_VER) && !defined(__clang__) + return _byteswap_uint64(n); + #elif defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(n); + #elif defined(__WATCOMC__) && defined(__386__) + return _watcom_bswap64(n); + #else + #error "This compiler does not support the byte swap intrinsic." + #endif +#else + /* Weird "<< 32" bitshift is required for C89 because it doesn't support 64-bit constants. Should be optimized out by a good compiler. */ + return ((n & ((drflac_uint64)0xFF000000 << 32)) >> 56) | + ((n & ((drflac_uint64)0x00FF0000 << 32)) >> 40) | + ((n & ((drflac_uint64)0x0000FF00 << 32)) >> 24) | + ((n & ((drflac_uint64)0x000000FF << 32)) >> 8) | + ((n & ((drflac_uint64)0xFF000000 )) << 8) | + ((n & ((drflac_uint64)0x00FF0000 )) << 24) | + ((n & ((drflac_uint64)0x0000FF00 )) << 40) | + ((n & ((drflac_uint64)0x000000FF )) << 56); +#endif +} + + +static DRFLAC_INLINE drflac_uint16 drflac__be2host_16(drflac_uint16 n) +{ + if (drflac__is_little_endian()) { + return drflac__swap_endian_uint16(n); + } + + return n; +} + +static DRFLAC_INLINE drflac_uint32 drflac__be2host_32(drflac_uint32 n) +{ + if (drflac__is_little_endian()) { + return drflac__swap_endian_uint32(n); + } + + return n; +} + +static DRFLAC_INLINE drflac_uint32 drflac__be2host_32_ptr_unaligned(const void* pData) +{ + const drflac_uint8* pNum = (drflac_uint8*)pData; + return *(pNum) << 24 | *(pNum+1) << 16 | *(pNum+2) << 8 | *(pNum+3); +} + +static DRFLAC_INLINE drflac_uint64 drflac__be2host_64(drflac_uint64 n) +{ + if (drflac__is_little_endian()) { + return drflac__swap_endian_uint64(n); + } + + return n; +} + + +static DRFLAC_INLINE drflac_uint32 drflac__le2host_32(drflac_uint32 n) +{ + if (!drflac__is_little_endian()) { + return drflac__swap_endian_uint32(n); + } + + return n; +} + +static DRFLAC_INLINE drflac_uint32 drflac__le2host_32_ptr_unaligned(const void* pData) +{ + const drflac_uint8* pNum = (drflac_uint8*)pData; + return *pNum | *(pNum+1) << 8 | *(pNum+2) << 16 | *(pNum+3) << 24; +} + + +static DRFLAC_INLINE drflac_uint32 drflac__unsynchsafe_32(drflac_uint32 n) +{ + drflac_uint32 result = 0; + result |= (n & 0x7F000000) >> 3; + result |= (n & 0x007F0000) >> 2; + result |= (n & 0x00007F00) >> 1; + result |= (n & 0x0000007F) >> 0; + + return result; +} + + + +/* The CRC code below is based on this document: http://zlib.net/crc_v3.txt */ +static drflac_uint8 drflac__crc8_table[] = { + 0x00, 0x07, 0x0E, 0x09, 0x1C, 0x1B, 0x12, 0x15, 0x38, 0x3F, 0x36, 0x31, 0x24, 0x23, 0x2A, 0x2D, + 0x70, 0x77, 0x7E, 0x79, 0x6C, 0x6B, 0x62, 0x65, 0x48, 0x4F, 0x46, 0x41, 0x54, 0x53, 0x5A, 0x5D, + 0xE0, 0xE7, 0xEE, 0xE9, 0xFC, 0xFB, 0xF2, 0xF5, 0xD8, 0xDF, 0xD6, 0xD1, 0xC4, 0xC3, 0xCA, 0xCD, + 0x90, 0x97, 0x9E, 0x99, 0x8C, 0x8B, 0x82, 0x85, 0xA8, 0xAF, 0xA6, 0xA1, 0xB4, 0xB3, 0xBA, 0xBD, + 0xC7, 0xC0, 0xC9, 0xCE, 0xDB, 0xDC, 0xD5, 0xD2, 0xFF, 0xF8, 0xF1, 0xF6, 0xE3, 0xE4, 0xED, 0xEA, + 0xB7, 0xB0, 0xB9, 0xBE, 0xAB, 0xAC, 0xA5, 0xA2, 0x8F, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9D, 0x9A, + 0x27, 0x20, 0x29, 0x2E, 0x3B, 0x3C, 0x35, 0x32, 0x1F, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0D, 0x0A, + 0x57, 0x50, 0x59, 0x5E, 0x4B, 0x4C, 0x45, 0x42, 0x6F, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7D, 0x7A, + 0x89, 0x8E, 0x87, 0x80, 0x95, 0x92, 0x9B, 0x9C, 0xB1, 0xB6, 0xBF, 0xB8, 0xAD, 0xAA, 0xA3, 0xA4, + 0xF9, 0xFE, 0xF7, 0xF0, 0xE5, 0xE2, 0xEB, 0xEC, 0xC1, 0xC6, 0xCF, 0xC8, 0xDD, 0xDA, 0xD3, 0xD4, + 0x69, 0x6E, 0x67, 0x60, 0x75, 0x72, 0x7B, 0x7C, 0x51, 0x56, 0x5F, 0x58, 0x4D, 0x4A, 0x43, 0x44, + 0x19, 0x1E, 0x17, 0x10, 0x05, 0x02, 0x0B, 0x0C, 0x21, 0x26, 0x2F, 0x28, 0x3D, 0x3A, 0x33, 0x34, + 0x4E, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5C, 0x5B, 0x76, 0x71, 0x78, 0x7F, 0x6A, 0x6D, 0x64, 0x63, + 0x3E, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2C, 0x2B, 0x06, 0x01, 0x08, 0x0F, 0x1A, 0x1D, 0x14, 0x13, + 0xAE, 0xA9, 0xA0, 0xA7, 0xB2, 0xB5, 0xBC, 0xBB, 0x96, 0x91, 0x98, 0x9F, 0x8A, 0x8D, 0x84, 0x83, + 0xDE, 0xD9, 0xD0, 0xD7, 0xC2, 0xC5, 0xCC, 0xCB, 0xE6, 0xE1, 0xE8, 0xEF, 0xFA, 0xFD, 0xF4, 0xF3 +}; + +static drflac_uint16 drflac__crc16_table[] = { + 0x0000, 0x8005, 0x800F, 0x000A, 0x801B, 0x001E, 0x0014, 0x8011, + 0x8033, 0x0036, 0x003C, 0x8039, 0x0028, 0x802D, 0x8027, 0x0022, + 0x8063, 0x0066, 0x006C, 0x8069, 0x0078, 0x807D, 0x8077, 0x0072, + 0x0050, 0x8055, 0x805F, 0x005A, 0x804B, 0x004E, 0x0044, 0x8041, + 0x80C3, 0x00C6, 0x00CC, 0x80C9, 0x00D8, 0x80DD, 0x80D7, 0x00D2, + 0x00F0, 0x80F5, 0x80FF, 0x00FA, 0x80EB, 0x00EE, 0x00E4, 0x80E1, + 0x00A0, 0x80A5, 0x80AF, 0x00AA, 0x80BB, 0x00BE, 0x00B4, 0x80B1, + 0x8093, 0x0096, 0x009C, 0x8099, 0x0088, 0x808D, 0x8087, 0x0082, + 0x8183, 0x0186, 0x018C, 0x8189, 0x0198, 0x819D, 0x8197, 0x0192, + 0x01B0, 0x81B5, 0x81BF, 0x01BA, 0x81AB, 0x01AE, 0x01A4, 0x81A1, + 0x01E0, 0x81E5, 0x81EF, 0x01EA, 0x81FB, 0x01FE, 0x01F4, 0x81F1, + 0x81D3, 0x01D6, 0x01DC, 0x81D9, 0x01C8, 0x81CD, 0x81C7, 0x01C2, + 0x0140, 0x8145, 0x814F, 0x014A, 0x815B, 0x015E, 0x0154, 0x8151, + 0x8173, 0x0176, 0x017C, 0x8179, 0x0168, 0x816D, 0x8167, 0x0162, + 0x8123, 0x0126, 0x012C, 0x8129, 0x0138, 0x813D, 0x8137, 0x0132, + 0x0110, 0x8115, 0x811F, 0x011A, 0x810B, 0x010E, 0x0104, 0x8101, + 0x8303, 0x0306, 0x030C, 0x8309, 0x0318, 0x831D, 0x8317, 0x0312, + 0x0330, 0x8335, 0x833F, 0x033A, 0x832B, 0x032E, 0x0324, 0x8321, + 0x0360, 0x8365, 0x836F, 0x036A, 0x837B, 0x037E, 0x0374, 0x8371, + 0x8353, 0x0356, 0x035C, 0x8359, 0x0348, 0x834D, 0x8347, 0x0342, + 0x03C0, 0x83C5, 0x83CF, 0x03CA, 0x83DB, 0x03DE, 0x03D4, 0x83D1, + 0x83F3, 0x03F6, 0x03FC, 0x83F9, 0x03E8, 0x83ED, 0x83E7, 0x03E2, + 0x83A3, 0x03A6, 0x03AC, 0x83A9, 0x03B8, 0x83BD, 0x83B7, 0x03B2, + 0x0390, 0x8395, 0x839F, 0x039A, 0x838B, 0x038E, 0x0384, 0x8381, + 0x0280, 0x8285, 0x828F, 0x028A, 0x829B, 0x029E, 0x0294, 0x8291, + 0x82B3, 0x02B6, 0x02BC, 0x82B9, 0x02A8, 0x82AD, 0x82A7, 0x02A2, + 0x82E3, 0x02E6, 0x02EC, 0x82E9, 0x02F8, 0x82FD, 0x82F7, 0x02F2, + 0x02D0, 0x82D5, 0x82DF, 0x02DA, 0x82CB, 0x02CE, 0x02C4, 0x82C1, + 0x8243, 0x0246, 0x024C, 0x8249, 0x0258, 0x825D, 0x8257, 0x0252, + 0x0270, 0x8275, 0x827F, 0x027A, 0x826B, 0x026E, 0x0264, 0x8261, + 0x0220, 0x8225, 0x822F, 0x022A, 0x823B, 0x023E, 0x0234, 0x8231, + 0x8213, 0x0216, 0x021C, 0x8219, 0x0208, 0x820D, 0x8207, 0x0202 +}; + +static DRFLAC_INLINE drflac_uint8 drflac_crc8_byte(drflac_uint8 crc, drflac_uint8 data) +{ + return drflac__crc8_table[crc ^ data]; +} + +static DRFLAC_INLINE drflac_uint8 drflac_crc8(drflac_uint8 crc, drflac_uint32 data, drflac_uint32 count) +{ +#ifdef DR_FLAC_NO_CRC + (void)crc; + (void)data; + (void)count; + return 0; +#else +#if 0 + /* REFERENCE (use of this implementation requires an explicit flush by doing "drflac_crc8(crc, 0, 8);") */ + drflac_uint8 p = 0x07; + for (int i = count-1; i >= 0; --i) { + drflac_uint8 bit = (data & (1 << i)) >> i; + if (crc & 0x80) { + crc = ((crc << 1) | bit) ^ p; + } else { + crc = ((crc << 1) | bit); + } + } + return crc; +#else + drflac_uint32 wholeBytes; + drflac_uint32 leftoverBits; + drflac_uint64 leftoverDataMask; + + static drflac_uint64 leftoverDataMaskTable[8] = { + 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F + }; + + DRFLAC_ASSERT(count <= 32); + + wholeBytes = count >> 3; + leftoverBits = count - (wholeBytes*8); + leftoverDataMask = leftoverDataMaskTable[leftoverBits]; + + switch (wholeBytes) { + case 4: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits))); + case 3: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits))); + case 2: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits))); + case 1: crc = drflac_crc8_byte(crc, (drflac_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits))); + case 0: if (leftoverBits > 0) crc = (drflac_uint8)((crc << leftoverBits) ^ drflac__crc8_table[(crc >> (8 - leftoverBits)) ^ (data & leftoverDataMask)]); + } + return crc; +#endif +#endif +} + +static DRFLAC_INLINE drflac_uint16 drflac_crc16_byte(drflac_uint16 crc, drflac_uint8 data) +{ + return (crc << 8) ^ drflac__crc16_table[(drflac_uint8)(crc >> 8) ^ data]; +} + +static DRFLAC_INLINE drflac_uint16 drflac_crc16_cache(drflac_uint16 crc, drflac_cache_t data) +{ +#ifdef DRFLAC_64BIT + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 56) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 48) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 40) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 32) & 0xFF)); +#endif + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 24) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 16) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 8) & 0xFF)); + crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 0) & 0xFF)); + + return crc; +} + +static DRFLAC_INLINE drflac_uint16 drflac_crc16_bytes(drflac_uint16 crc, drflac_cache_t data, drflac_uint32 byteCount) +{ + switch (byteCount) + { +#ifdef DRFLAC_64BIT + case 8: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 56) & 0xFF)); + case 7: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 48) & 0xFF)); + case 6: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 40) & 0xFF)); + case 5: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 32) & 0xFF)); +#endif + case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 24) & 0xFF)); + case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 16) & 0xFF)); + case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 8) & 0xFF)); + case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data >> 0) & 0xFF)); + } + + return crc; +} + +#if 0 +static DRFLAC_INLINE drflac_uint16 drflac_crc16__32bit(drflac_uint16 crc, drflac_uint32 data, drflac_uint32 count) +{ +#ifdef DR_FLAC_NO_CRC + (void)crc; + (void)data; + (void)count; + return 0; +#else +#if 0 + /* REFERENCE (use of this implementation requires an explicit flush by doing "drflac_crc16(crc, 0, 16);") */ + drflac_uint16 p = 0x8005; + for (int i = count-1; i >= 0; --i) { + drflac_uint16 bit = (data & (1ULL << i)) >> i; + if (r & 0x8000) { + r = ((r << 1) | bit) ^ p; + } else { + r = ((r << 1) | bit); + } + } + + return crc; +#else + drflac_uint32 wholeBytes; + drflac_uint32 leftoverBits; + drflac_uint64 leftoverDataMask; + + static drflac_uint64 leftoverDataMaskTable[8] = { + 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F + }; + + DRFLAC_ASSERT(count <= 64); + + wholeBytes = count >> 3; + leftoverBits = count & 7; + leftoverDataMask = leftoverDataMaskTable[leftoverBits]; + + switch (wholeBytes) { + default: + case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits))); + case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits))); + case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits))); + case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits))); + case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ drflac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)]; + } + return crc; +#endif +#endif +} + +static DRFLAC_INLINE drflac_uint16 drflac_crc16__64bit(drflac_uint16 crc, drflac_uint64 data, drflac_uint32 count) +{ +#ifdef DR_FLAC_NO_CRC + (void)crc; + (void)data; + (void)count; + return 0; +#else + drflac_uint32 wholeBytes; + drflac_uint32 leftoverBits; + drflac_uint64 leftoverDataMask; + + static drflac_uint64 leftoverDataMaskTable[8] = { + 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F + }; + + DRFLAC_ASSERT(count <= 64); + + wholeBytes = count >> 3; + leftoverBits = count & 7; + leftoverDataMask = leftoverDataMaskTable[leftoverBits]; + + switch (wholeBytes) { + default: + case 8: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0xFF000000 << 32) << leftoverBits)) >> (56 + leftoverBits))); /* Weird "<< 32" bitshift is required for C89 because it doesn't support 64-bit constants. Should be optimized out by a good compiler. */ + case 7: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x00FF0000 << 32) << leftoverBits)) >> (48 + leftoverBits))); + case 6: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x0000FF00 << 32) << leftoverBits)) >> (40 + leftoverBits))); + case 5: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x000000FF << 32) << leftoverBits)) >> (32 + leftoverBits))); + case 4: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0xFF000000 ) << leftoverBits)) >> (24 + leftoverBits))); + case 3: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x00FF0000 ) << leftoverBits)) >> (16 + leftoverBits))); + case 2: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x0000FF00 ) << leftoverBits)) >> ( 8 + leftoverBits))); + case 1: crc = drflac_crc16_byte(crc, (drflac_uint8)((data & (((drflac_uint64)0x000000FF ) << leftoverBits)) >> ( 0 + leftoverBits))); + case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ drflac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)]; + } + return crc; +#endif +} + + +static DRFLAC_INLINE drflac_uint16 drflac_crc16(drflac_uint16 crc, drflac_cache_t data, drflac_uint32 count) +{ +#ifdef DRFLAC_64BIT + return drflac_crc16__64bit(crc, data, count); +#else + return drflac_crc16__32bit(crc, data, count); +#endif +} +#endif + + +#ifdef DRFLAC_64BIT +#define drflac__be2host__cache_line drflac__be2host_64 +#else +#define drflac__be2host__cache_line drflac__be2host_32 +#endif + +/* +BIT READING ATTEMPT #2 + +This uses a 32- or 64-bit bit-shifted cache - as bits are read, the cache is shifted such that the first valid bit is sitting +on the most significant bit. It uses the notion of an L1 and L2 cache (borrowed from CPU architecture), where the L1 cache +is a 32- or 64-bit unsigned integer (depending on whether or not a 32- or 64-bit build is being compiled) and the L2 is an +array of "cache lines", with each cache line being the same size as the L1. The L2 is a buffer of about 4KB and is where data +from onRead() is read into. +*/ +#define DRFLAC_CACHE_L1_SIZE_BYTES(bs) (sizeof((bs)->cache)) +#define DRFLAC_CACHE_L1_SIZE_BITS(bs) (sizeof((bs)->cache)*8) +#define DRFLAC_CACHE_L1_BITS_REMAINING(bs) (DRFLAC_CACHE_L1_SIZE_BITS(bs) - (bs)->consumedBits) +#define DRFLAC_CACHE_L1_SELECTION_MASK(_bitCount) (~((~(drflac_cache_t)0) >> (_bitCount))) +#define DRFLAC_CACHE_L1_SELECTION_SHIFT(bs, _bitCount) (DRFLAC_CACHE_L1_SIZE_BITS(bs) - (_bitCount)) +#define DRFLAC_CACHE_L1_SELECT(bs, _bitCount) (((bs)->cache) & DRFLAC_CACHE_L1_SELECTION_MASK(_bitCount)) +#define DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, _bitCount) (DRFLAC_CACHE_L1_SELECT((bs), (_bitCount)) >> DRFLAC_CACHE_L1_SELECTION_SHIFT((bs), (_bitCount))) +#define DRFLAC_CACHE_L1_SELECT_AND_SHIFT_SAFE(bs, _bitCount)(DRFLAC_CACHE_L1_SELECT((bs), (_bitCount)) >> (DRFLAC_CACHE_L1_SELECTION_SHIFT((bs), (_bitCount)) & (DRFLAC_CACHE_L1_SIZE_BITS(bs)-1))) +#define DRFLAC_CACHE_L2_SIZE_BYTES(bs) (sizeof((bs)->cacheL2)) +#define DRFLAC_CACHE_L2_LINE_COUNT(bs) (DRFLAC_CACHE_L2_SIZE_BYTES(bs) / sizeof((bs)->cacheL2[0])) +#define DRFLAC_CACHE_L2_LINES_REMAINING(bs) (DRFLAC_CACHE_L2_LINE_COUNT(bs) - (bs)->nextL2Line) + + +#ifndef DR_FLAC_NO_CRC +static DRFLAC_INLINE void drflac__reset_crc16(drflac_bs* bs) +{ + bs->crc16 = 0; + bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; +} + +static DRFLAC_INLINE void drflac__update_crc16(drflac_bs* bs) +{ + if (bs->crc16CacheIgnoredBytes == 0) { + bs->crc16 = drflac_crc16_cache(bs->crc16, bs->crc16Cache); + } else { + bs->crc16 = drflac_crc16_bytes(bs->crc16, bs->crc16Cache, DRFLAC_CACHE_L1_SIZE_BYTES(bs) - bs->crc16CacheIgnoredBytes); + bs->crc16CacheIgnoredBytes = 0; + } +} + +static DRFLAC_INLINE drflac_uint16 drflac__flush_crc16(drflac_bs* bs) +{ + /* We should never be flushing in a situation where we are not aligned on a byte boundary. */ + DRFLAC_ASSERT((DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7) == 0); + + /* + The bits that were read from the L1 cache need to be accumulated. The number of bytes needing to be accumulated is determined + by the number of bits that have been consumed. + */ + if (DRFLAC_CACHE_L1_BITS_REMAINING(bs) == 0) { + drflac__update_crc16(bs); + } else { + /* We only accumulate the consumed bits. */ + bs->crc16 = drflac_crc16_bytes(bs->crc16, bs->crc16Cache >> DRFLAC_CACHE_L1_BITS_REMAINING(bs), (bs->consumedBits >> 3) - bs->crc16CacheIgnoredBytes); + + /* + The bits that we just accumulated should never be accumulated again. We need to keep track of how many bytes were accumulated + so we can handle that later. + */ + bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; + } + + return bs->crc16; +} +#endif + +static DRFLAC_INLINE drflac_bool32 drflac__reload_l1_cache_from_l2(drflac_bs* bs) +{ + size_t bytesRead; + size_t alignedL1LineCount; + + /* Fast path. Try loading straight from L2. */ + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { + bs->cache = bs->cacheL2[bs->nextL2Line++]; + return DRFLAC_TRUE; + } + + /* + If we get here it means we've run out of data in the L2 cache. We'll need to fetch more from the client, if there's + any left. + */ + if (bs->unalignedByteCount > 0) { + return DRFLAC_FALSE; /* If we have any unaligned bytes it means there's no more aligned bytes left in the client. */ + } + + bytesRead = bs->onRead(bs->pUserData, bs->cacheL2, DRFLAC_CACHE_L2_SIZE_BYTES(bs)); + + bs->nextL2Line = 0; + if (bytesRead == DRFLAC_CACHE_L2_SIZE_BYTES(bs)) { + bs->cache = bs->cacheL2[bs->nextL2Line++]; + return DRFLAC_TRUE; + } + + + /* + If we get here it means we were unable to retrieve enough data to fill the entire L2 cache. It probably + means we've just reached the end of the file. We need to move the valid data down to the end of the buffer + and adjust the index of the next line accordingly. Also keep in mind that the L2 cache must be aligned to + the size of the L1 so we'll need to seek backwards by any misaligned bytes. + */ + alignedL1LineCount = bytesRead / DRFLAC_CACHE_L1_SIZE_BYTES(bs); + + /* We need to keep track of any unaligned bytes for later use. */ + bs->unalignedByteCount = bytesRead - (alignedL1LineCount * DRFLAC_CACHE_L1_SIZE_BYTES(bs)); + if (bs->unalignedByteCount > 0) { + bs->unalignedCache = bs->cacheL2[alignedL1LineCount]; + } + + if (alignedL1LineCount > 0) { + size_t offset = DRFLAC_CACHE_L2_LINE_COUNT(bs) - alignedL1LineCount; + size_t i; + for (i = alignedL1LineCount; i > 0; --i) { + bs->cacheL2[i-1 + offset] = bs->cacheL2[i-1]; + } + + bs->nextL2Line = (drflac_uint32)offset; + bs->cache = bs->cacheL2[bs->nextL2Line++]; + return DRFLAC_TRUE; + } else { + /* If we get into this branch it means we weren't able to load any L1-aligned data. */ + bs->nextL2Line = DRFLAC_CACHE_L2_LINE_COUNT(bs); + return DRFLAC_FALSE; + } +} + +static drflac_bool32 drflac__reload_cache(drflac_bs* bs) +{ + size_t bytesRead; + +#ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); +#endif + + /* Fast path. Try just moving the next value in the L2 cache to the L1 cache. */ + if (drflac__reload_l1_cache_from_l2(bs)) { + bs->cache = drflac__be2host__cache_line(bs->cache); + bs->consumedBits = 0; +#ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs->cache; +#endif + return DRFLAC_TRUE; + } + + /* Slow path. */ + + /* + If we get here it means we have failed to load the L1 cache from the L2. Likely we've just reached the end of the stream and the last + few bytes did not meet the alignment requirements for the L2 cache. In this case we need to fall back to a slower path and read the + data from the unaligned cache. + */ + bytesRead = bs->unalignedByteCount; + if (bytesRead == 0) { + bs->consumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs); /* <-- The stream has been exhausted, so marked the bits as consumed. */ + return DRFLAC_FALSE; + } + + DRFLAC_ASSERT(bytesRead < DRFLAC_CACHE_L1_SIZE_BYTES(bs)); + bs->consumedBits = (drflac_uint32)(DRFLAC_CACHE_L1_SIZE_BYTES(bs) - bytesRead) * 8; + + bs->cache = drflac__be2host__cache_line(bs->unalignedCache); + bs->cache &= DRFLAC_CACHE_L1_SELECTION_MASK(DRFLAC_CACHE_L1_BITS_REMAINING(bs)); /* <-- Make sure the consumed bits are always set to zero. Other parts of the library depend on this property. */ + bs->unalignedByteCount = 0; /* <-- At this point the unaligned bytes have been moved into the cache and we thus have no more unaligned bytes. */ + +#ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs->cache >> bs->consumedBits; + bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3; +#endif + return DRFLAC_TRUE; +} + +static void drflac__reset_cache(drflac_bs* bs) +{ + bs->nextL2Line = DRFLAC_CACHE_L2_LINE_COUNT(bs); /* <-- This clears the L2 cache. */ + bs->consumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs); /* <-- This clears the L1 cache. */ + bs->cache = 0; + bs->unalignedByteCount = 0; /* <-- This clears the trailing unaligned bytes. */ + bs->unalignedCache = 0; + +#ifndef DR_FLAC_NO_CRC + bs->crc16Cache = 0; + bs->crc16CacheIgnoredBytes = 0; +#endif +} + + +static DRFLAC_INLINE drflac_bool32 drflac__read_uint32(drflac_bs* bs, unsigned int bitCount, drflac_uint32* pResultOut) +{ + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResultOut != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 32); + + if (bs->consumedBits == DRFLAC_CACHE_L1_SIZE_BITS(bs)) { + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + } + + if (bitCount <= DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* + If we want to load all 32-bits from a 32-bit cache we need to do it slightly differently because we can't do + a 32-bit shift on a 32-bit integer. This will never be the case on 64-bit caches, so we can have a slightly + more optimal solution for this. + */ +#ifdef DRFLAC_64BIT + *pResultOut = (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCount); + bs->consumedBits += bitCount; + bs->cache <<= bitCount; +#else + if (bitCount < DRFLAC_CACHE_L1_SIZE_BITS(bs)) { + *pResultOut = (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCount); + bs->consumedBits += bitCount; + bs->cache <<= bitCount; + } else { + /* Cannot shift by 32-bits, so need to do it differently. */ + *pResultOut = (drflac_uint32)bs->cache; + bs->consumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs); + bs->cache = 0; + } +#endif + + return DRFLAC_TRUE; + } else { + /* It straddles the cached data. It will never cover more than the next chunk. We just read the number in two parts and combine them. */ + drflac_uint32 bitCountHi = DRFLAC_CACHE_L1_BITS_REMAINING(bs); + drflac_uint32 bitCountLo = bitCount - bitCountHi; + drflac_uint32 resultHi; + + DRFLAC_ASSERT(bitCountHi > 0); + DRFLAC_ASSERT(bitCountHi < 32); + resultHi = (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountHi); + + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + if (bitCountLo > DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* This happens when we get to end of stream */ + return DRFLAC_FALSE; + } + + *pResultOut = (resultHi << bitCountLo) | (drflac_uint32)DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountLo); + bs->consumedBits += bitCountLo; + bs->cache <<= bitCountLo; + return DRFLAC_TRUE; + } +} + +static drflac_bool32 drflac__read_int32(drflac_bs* bs, unsigned int bitCount, drflac_int32* pResult) +{ + drflac_uint32 result; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResult != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 32); + + if (!drflac__read_uint32(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + /* Do not attempt to shift by 32 as it's undefined. */ + if (bitCount < 32) { + drflac_uint32 signbit; + signbit = ((result >> (bitCount-1)) & 0x01); + result |= (~signbit + 1) << bitCount; + } + + *pResult = (drflac_int32)result; + return DRFLAC_TRUE; +} + +#ifdef DRFLAC_64BIT +static drflac_bool32 drflac__read_uint64(drflac_bs* bs, unsigned int bitCount, drflac_uint64* pResultOut) +{ + drflac_uint32 resultHi; + drflac_uint32 resultLo; + + DRFLAC_ASSERT(bitCount <= 64); + DRFLAC_ASSERT(bitCount > 32); + + if (!drflac__read_uint32(bs, bitCount - 32, &resultHi)) { + return DRFLAC_FALSE; + } + + if (!drflac__read_uint32(bs, 32, &resultLo)) { + return DRFLAC_FALSE; + } + + *pResultOut = (((drflac_uint64)resultHi) << 32) | ((drflac_uint64)resultLo); + return DRFLAC_TRUE; +} +#endif + +/* Function below is unused, but leaving it here in case I need to quickly add it again. */ +#if 0 +static drflac_bool32 drflac__read_int64(drflac_bs* bs, unsigned int bitCount, drflac_int64* pResultOut) +{ + drflac_uint64 result; + drflac_uint64 signbit; + + DRFLAC_ASSERT(bitCount <= 64); + + if (!drflac__read_uint64(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + signbit = ((result >> (bitCount-1)) & 0x01); + result |= (~signbit + 1) << bitCount; + + *pResultOut = (drflac_int64)result; + return DRFLAC_TRUE; +} +#endif + +static drflac_bool32 drflac__read_uint16(drflac_bs* bs, unsigned int bitCount, drflac_uint16* pResult) +{ + drflac_uint32 result; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResult != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 16); + + if (!drflac__read_uint32(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + *pResult = (drflac_uint16)result; + return DRFLAC_TRUE; +} + +#if 0 +static drflac_bool32 drflac__read_int16(drflac_bs* bs, unsigned int bitCount, drflac_int16* pResult) +{ + drflac_int32 result; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResult != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 16); + + if (!drflac__read_int32(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + *pResult = (drflac_int16)result; + return DRFLAC_TRUE; +} +#endif + +static drflac_bool32 drflac__read_uint8(drflac_bs* bs, unsigned int bitCount, drflac_uint8* pResult) +{ + drflac_uint32 result; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResult != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 8); + + if (!drflac__read_uint32(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + *pResult = (drflac_uint8)result; + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__read_int8(drflac_bs* bs, unsigned int bitCount, drflac_int8* pResult) +{ + drflac_int32 result; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pResult != NULL); + DRFLAC_ASSERT(bitCount > 0); + DRFLAC_ASSERT(bitCount <= 8); + + if (!drflac__read_int32(bs, bitCount, &result)) { + return DRFLAC_FALSE; + } + + *pResult = (drflac_int8)result; + return DRFLAC_TRUE; +} + + +static drflac_bool32 drflac__seek_bits(drflac_bs* bs, size_t bitsToSeek) +{ + if (bitsToSeek <= DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + bs->consumedBits += (drflac_uint32)bitsToSeek; + bs->cache <<= bitsToSeek; + return DRFLAC_TRUE; + } else { + /* It straddles the cached data. This function isn't called too frequently so I'm favouring simplicity here. */ + bitsToSeek -= DRFLAC_CACHE_L1_BITS_REMAINING(bs); + bs->consumedBits += DRFLAC_CACHE_L1_BITS_REMAINING(bs); + bs->cache = 0; + + /* Simple case. Seek in groups of the same number as bits that fit within a cache line. */ +#ifdef DRFLAC_64BIT + while (bitsToSeek >= DRFLAC_CACHE_L1_SIZE_BITS(bs)) { + drflac_uint64 bin; + if (!drflac__read_uint64(bs, DRFLAC_CACHE_L1_SIZE_BITS(bs), &bin)) { + return DRFLAC_FALSE; + } + bitsToSeek -= DRFLAC_CACHE_L1_SIZE_BITS(bs); + } +#else + while (bitsToSeek >= DRFLAC_CACHE_L1_SIZE_BITS(bs)) { + drflac_uint32 bin; + if (!drflac__read_uint32(bs, DRFLAC_CACHE_L1_SIZE_BITS(bs), &bin)) { + return DRFLAC_FALSE; + } + bitsToSeek -= DRFLAC_CACHE_L1_SIZE_BITS(bs); + } +#endif + + /* Whole leftover bytes. */ + while (bitsToSeek >= 8) { + drflac_uint8 bin; + if (!drflac__read_uint8(bs, 8, &bin)) { + return DRFLAC_FALSE; + } + bitsToSeek -= 8; + } + + /* Leftover bits. */ + if (bitsToSeek > 0) { + drflac_uint8 bin; + if (!drflac__read_uint8(bs, (drflac_uint32)bitsToSeek, &bin)) { + return DRFLAC_FALSE; + } + bitsToSeek = 0; /* <-- Necessary for the assert below. */ + } + + DRFLAC_ASSERT(bitsToSeek == 0); + return DRFLAC_TRUE; + } +} + + +/* This function moves the bit streamer to the first bit after the sync code (bit 15 of the of the frame header). It will also update the CRC-16. */ +static drflac_bool32 drflac__find_and_seek_to_next_sync_code(drflac_bs* bs) +{ + DRFLAC_ASSERT(bs != NULL); + + /* + The sync code is always aligned to 8 bits. This is convenient for us because it means we can do byte-aligned movements. The first + thing to do is align to the next byte. + */ + if (!drflac__seek_bits(bs, DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) { + return DRFLAC_FALSE; + } + + for (;;) { + drflac_uint8 hi; + +#ifndef DR_FLAC_NO_CRC + drflac__reset_crc16(bs); +#endif + + if (!drflac__read_uint8(bs, 8, &hi)) { + return DRFLAC_FALSE; + } + + if (hi == 0xFF) { + drflac_uint8 lo; + if (!drflac__read_uint8(bs, 6, &lo)) { + return DRFLAC_FALSE; + } + + if (lo == 0x3E) { + return DRFLAC_TRUE; + } else { + if (!drflac__seek_bits(bs, DRFLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) { + return DRFLAC_FALSE; + } + } + } + } + + /* Should never get here. */ + /*return DRFLAC_FALSE;*/ +} + + +#if defined(DRFLAC_HAS_LZCNT_INTRINSIC) +#define DRFLAC_IMPLEMENT_CLZ_LZCNT +#endif +#if defined(_MSC_VER) && _MSC_VER >= 1400 && (defined(DRFLAC_X64) || defined(DRFLAC_X86)) && !defined(__clang__) +#define DRFLAC_IMPLEMENT_CLZ_MSVC +#endif +#if defined(__WATCOMC__) && defined(__386__) +#define DRFLAC_IMPLEMENT_CLZ_WATCOM +#endif +#ifdef __MRC__ +#include +#define DRFLAC_IMPLEMENT_CLZ_MRC +#endif + +static DRFLAC_INLINE drflac_uint32 drflac__clz_software(drflac_cache_t x) +{ + drflac_uint32 n; + static drflac_uint32 clz_table_4[] = { + 0, + 4, + 3, 3, + 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + + if (x == 0) { + return sizeof(x)*8; + } + + n = clz_table_4[x >> (sizeof(x)*8 - 4)]; + if (n == 0) { +#ifdef DRFLAC_64BIT + if ((x & ((drflac_uint64)0xFFFFFFFF << 32)) == 0) { n = 32; x <<= 32; } + if ((x & ((drflac_uint64)0xFFFF0000 << 32)) == 0) { n += 16; x <<= 16; } + if ((x & ((drflac_uint64)0xFF000000 << 32)) == 0) { n += 8; x <<= 8; } + if ((x & ((drflac_uint64)0xF0000000 << 32)) == 0) { n += 4; x <<= 4; } +#else + if ((x & 0xFFFF0000) == 0) { n = 16; x <<= 16; } + if ((x & 0xFF000000) == 0) { n += 8; x <<= 8; } + if ((x & 0xF0000000) == 0) { n += 4; x <<= 4; } +#endif + n += clz_table_4[x >> (sizeof(x)*8 - 4)]; + } + + return n - 1; +} + +#ifdef DRFLAC_IMPLEMENT_CLZ_LZCNT +static DRFLAC_INLINE drflac_bool32 drflac__is_lzcnt_supported(void) +{ + /* Fast compile time check for ARM. */ +#if defined(DRFLAC_HAS_LZCNT_INTRINSIC) && defined(DRFLAC_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5) + return DRFLAC_TRUE; +#elif defined(__MRC__) + return DRFLAC_TRUE; +#else + /* If the compiler itself does not support the intrinsic then we'll need to return false. */ + #ifdef DRFLAC_HAS_LZCNT_INTRINSIC + return drflac__gIsLZCNTSupported; + #else + return DRFLAC_FALSE; + #endif +#endif +} + +static DRFLAC_INLINE drflac_uint32 drflac__clz_lzcnt(drflac_cache_t x) +{ + /* + It's critical for competitive decoding performance that this function be highly optimal. With MSVC we can use the __lzcnt64() and __lzcnt() intrinsics + to achieve good performance, however on GCC and Clang it's a little bit more annoying. The __builtin_clzl() and __builtin_clzll() intrinsics leave + it undefined as to the return value when `x` is 0. We need this to be well defined as returning 32 or 64, depending on whether or not it's a 32- or + 64-bit build. To work around this we would need to add a conditional to check for the x = 0 case, but this creates unnecessary inefficiency. To work + around this problem I have written some inline assembly to emit the LZCNT (x86) or CLZ (ARM) instruction directly which removes the need to include + the conditional. This has worked well in the past, but for some reason Clang's MSVC compatible driver, clang-cl, does not seem to be handling this + in the same way as the normal Clang driver. It seems that `clang-cl` is just outputting the wrong results sometimes, maybe due to some register + getting clobbered? + + I'm not sure if this is a bug with dr_flac's inlined assembly (most likely), a bug in `clang-cl` or just a misunderstanding on my part with inline + assembly rules for `clang-cl`. If somebody can identify an error in dr_flac's inlined assembly I'm happy to get that fixed. + + Fortunately there is an easy workaround for this. Clang implements MSVC-specific intrinsics for compatibility. It also defines _MSC_VER for extra + compatibility. We can therefore just check for _MSC_VER and use the MSVC intrinsic which, fortunately for us, Clang supports. It would still be nice + to know how to fix the inlined assembly for correctness sake, however. + */ + +#if defined(_MSC_VER) /*&& !defined(__clang__)*/ /* <-- Intentionally wanting Clang to use the MSVC __lzcnt64/__lzcnt intrinsics due to above ^. */ + #ifdef DRFLAC_64BIT + return (drflac_uint32)__lzcnt64(x); + #else + return (drflac_uint32)__lzcnt(x); + #endif +#else + #if defined(__GNUC__) || defined(__clang__) + #if defined(DRFLAC_X64) + { + drflac_uint64 r; + __asm__ __volatile__ ( + "lzcnt{ %1, %0| %0, %1}" : "=r"(r) : "r"(x) : "cc" + ); + + return (drflac_uint32)r; + } + #elif defined(DRFLAC_X86) + { + drflac_uint32 r; + __asm__ __volatile__ ( + "lzcnt{l %1, %0| %0, %1}" : "=r"(r) : "r"(x) : "cc" + ); + + return r; + } + #elif defined(DRFLAC_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5) && !defined(__ARM_ARCH_6M__) && !defined(DRFLAC_64BIT) /* <-- I haven't tested 64-bit inline assembly, so only enabling this for the 32-bit build for now. */ + { + unsigned int r; + __asm__ __volatile__ ( + #if defined(DRFLAC_64BIT) + "clz %w[out], %w[in]" : [out]"=r"(r) : [in]"r"(x) /* <-- This is untested. If someone in the community could test this, that would be appreciated! */ + #else + "clz %[out], %[in]" : [out]"=r"(r) : [in]"r"(x) + #endif + ); + + return r; + } + #else + if (x == 0) { + return sizeof(x)*8; + } + #ifdef DRFLAC_64BIT + return (drflac_uint32)__builtin_clzll((drflac_uint64)x); + #else + return (drflac_uint32)__builtin_clzl((drflac_uint32)x); + #endif + #endif + #else + /* Unsupported compiler. */ + #error "This compiler does not support the lzcnt intrinsic." + #endif +#endif +} +#endif + +#ifdef DRFLAC_IMPLEMENT_CLZ_MSVC +#include /* For BitScanReverse(). */ + +static DRFLAC_INLINE drflac_uint32 drflac__clz_msvc(drflac_cache_t x) +{ + drflac_uint32 n; + + if (x == 0) { + return sizeof(x)*8; + } + +#ifdef DRFLAC_64BIT + _BitScanReverse64((unsigned long*)&n, x); +#else + _BitScanReverse((unsigned long*)&n, x); +#endif + return sizeof(x)*8 - n - 1; +} +#endif + +#ifdef DRFLAC_IMPLEMENT_CLZ_WATCOM +static __inline drflac_uint32 drflac__clz_watcom (drflac_uint32); +#ifdef DRFLAC_IMPLEMENT_CLZ_WATCOM_LZCNT +/* Use the LZCNT instruction (only available on some processors since the 2010s). */ +#pragma aux drflac__clz_watcom_lzcnt = \ + "db 0F3h, 0Fh, 0BDh, 0C0h" /* lzcnt eax, eax */ \ + parm [eax] \ + value [eax] \ + modify nomemory; +#else +/* Use the 386+-compatible implementation. */ +#pragma aux drflac__clz_watcom = \ + "bsr eax, eax" \ + "xor eax, 31" \ + parm [eax] nomemory \ + value [eax] \ + modify exact [eax] nomemory; +#endif +#endif + +static DRFLAC_INLINE drflac_uint32 drflac__clz(drflac_cache_t x) +{ +#ifdef DRFLAC_IMPLEMENT_CLZ_LZCNT + if (drflac__is_lzcnt_supported()) { + return drflac__clz_lzcnt(x); + } else +#endif + { +#ifdef DRFLAC_IMPLEMENT_CLZ_MSVC + return drflac__clz_msvc(x); +#elif defined(DRFLAC_IMPLEMENT_CLZ_WATCOM_LZCNT) + return drflac__clz_watcom_lzcnt(x); +#elif defined(DRFLAC_IMPLEMENT_CLZ_WATCOM) + return (x == 0) ? sizeof(x)*8 : drflac__clz_watcom(x); +#elif defined(__MRC__) + return __cntlzw(x); +#else + return drflac__clz_software(x); +#endif + } +} + + +static DRFLAC_INLINE drflac_bool32 drflac__seek_past_next_set_bit(drflac_bs* bs, unsigned int* pOffsetOut) +{ + drflac_uint32 zeroCounter = 0; + drflac_uint32 setBitOffsetPlus1; + + while (bs->cache == 0) { + zeroCounter += (drflac_uint32)DRFLAC_CACHE_L1_BITS_REMAINING(bs); + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + } + + if (bs->cache == 1) { + /* Not catching this would lead to undefined behaviour: a shift of a 32-bit number by 32 or more is undefined */ + *pOffsetOut = zeroCounter + (drflac_uint32)DRFLAC_CACHE_L1_BITS_REMAINING(bs) - 1; + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + + return DRFLAC_TRUE; + } + + setBitOffsetPlus1 = drflac__clz(bs->cache); + setBitOffsetPlus1 += 1; + + if (setBitOffsetPlus1 > DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* This happens when we get to end of stream */ + return DRFLAC_FALSE; + } + + bs->consumedBits += setBitOffsetPlus1; + bs->cache <<= setBitOffsetPlus1; + + *pOffsetOut = zeroCounter + setBitOffsetPlus1 - 1; + return DRFLAC_TRUE; +} + + + +static drflac_bool32 drflac__seek_to_byte(drflac_bs* bs, drflac_uint64 offsetFromStart) +{ + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(offsetFromStart > 0); + + /* + Seeking from the start is not quite as trivial as it sounds because the onSeek callback takes a signed 32-bit integer (which + is intentional because it simplifies the implementation of the onSeek callbacks), however offsetFromStart is unsigned 64-bit. + To resolve we just need to do an initial seek from the start, and then a series of offset seeks to make up the remainder. + */ + if (offsetFromStart > 0x7FFFFFFF) { + drflac_uint64 bytesRemaining = offsetFromStart; + if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + bytesRemaining -= 0x7FFFFFFF; + + while (bytesRemaining > 0x7FFFFFFF) { + if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + bytesRemaining -= 0x7FFFFFFF; + } + + if (bytesRemaining > 0) { + if (!bs->onSeek(bs->pUserData, (int)bytesRemaining, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + } + } else { + if (!bs->onSeek(bs->pUserData, (int)offsetFromStart, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + } + + /* The cache should be reset to force a reload of fresh data from the client. */ + drflac__reset_cache(bs); + return DRFLAC_TRUE; +} + + +static drflac_result drflac__read_utf8_coded_number(drflac_bs* bs, drflac_uint64* pNumberOut, drflac_uint8* pCRCOut) +{ + drflac_uint8 crc; + drflac_uint64 result; + drflac_uint8 utf8[7] = {0}; + int byteCount; + int i; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pNumberOut != NULL); + DRFLAC_ASSERT(pCRCOut != NULL); + + crc = *pCRCOut; + + if (!drflac__read_uint8(bs, 8, utf8)) { + *pNumberOut = 0; + return DRFLAC_AT_END; + } + crc = drflac_crc8(crc, utf8[0], 8); + + if ((utf8[0] & 0x80) == 0) { + *pNumberOut = utf8[0]; + *pCRCOut = crc; + return DRFLAC_SUCCESS; + } + + /*byteCount = 1;*/ + if ((utf8[0] & 0xE0) == 0xC0) { + byteCount = 2; + } else if ((utf8[0] & 0xF0) == 0xE0) { + byteCount = 3; + } else if ((utf8[0] & 0xF8) == 0xF0) { + byteCount = 4; + } else if ((utf8[0] & 0xFC) == 0xF8) { + byteCount = 5; + } else if ((utf8[0] & 0xFE) == 0xFC) { + byteCount = 6; + } else if ((utf8[0] & 0xFF) == 0xFE) { + byteCount = 7; + } else { + *pNumberOut = 0; + return DRFLAC_CRC_MISMATCH; /* Bad UTF-8 encoding. */ + } + + /* Read extra bytes. */ + DRFLAC_ASSERT(byteCount > 1); + + result = (drflac_uint64)(utf8[0] & (0xFF >> (byteCount + 1))); + for (i = 1; i < byteCount; ++i) { + if (!drflac__read_uint8(bs, 8, utf8 + i)) { + *pNumberOut = 0; + return DRFLAC_AT_END; + } + crc = drflac_crc8(crc, utf8[i], 8); + + result = (result << 6) | (utf8[i] & 0x3F); + } + + *pNumberOut = result; + *pCRCOut = crc; + return DRFLAC_SUCCESS; +} + + +static DRFLAC_INLINE drflac_uint32 drflac__ilog2_u32(drflac_uint32 x) +{ +#if 1 /* Needs optimizing. */ + drflac_uint32 result = 0; + while (x > 0) { + result += 1; + x >>= 1; + } + + return result; +#endif +} + +static DRFLAC_INLINE drflac_bool32 drflac__use_64_bit_prediction(drflac_uint32 bitsPerSample, drflac_uint32 order, drflac_uint32 precision) +{ + /* https://web.archive.org/web/20220205005724/https://github.com/ietf-wg-cellar/flac-specification/blob/37a49aa48ba4ba12e8757badfc59c0df35435fec/rfc_backmatter.md */ + return bitsPerSample + precision + drflac__ilog2_u32(order) > 32; +} + + +/* +The next two functions are responsible for calculating the prediction. + +When the bits per sample is >16 we need to use 64-bit integer arithmetic because otherwise we'll run out of precision. It's +safe to assume this will be slower on 32-bit platforms so we use a more optimal solution when the bits per sample is <=16. +*/ +#if defined(__clang__) +__attribute__((no_sanitize("signed-integer-overflow"))) +#endif +static DRFLAC_INLINE drflac_int32 drflac__calculate_prediction_32(drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) +{ + drflac_int32 prediction = 0; + + DRFLAC_ASSERT(order <= 32); + + /* 32-bit version. */ + + /* VC++ optimizes this to a single jmp. I've not yet verified this for other compilers. */ + switch (order) + { + case 32: prediction += coefficients[31] * pDecodedSamples[-32]; + case 31: prediction += coefficients[30] * pDecodedSamples[-31]; + case 30: prediction += coefficients[29] * pDecodedSamples[-30]; + case 29: prediction += coefficients[28] * pDecodedSamples[-29]; + case 28: prediction += coefficients[27] * pDecodedSamples[-28]; + case 27: prediction += coefficients[26] * pDecodedSamples[-27]; + case 26: prediction += coefficients[25] * pDecodedSamples[-26]; + case 25: prediction += coefficients[24] * pDecodedSamples[-25]; + case 24: prediction += coefficients[23] * pDecodedSamples[-24]; + case 23: prediction += coefficients[22] * pDecodedSamples[-23]; + case 22: prediction += coefficients[21] * pDecodedSamples[-22]; + case 21: prediction += coefficients[20] * pDecodedSamples[-21]; + case 20: prediction += coefficients[19] * pDecodedSamples[-20]; + case 19: prediction += coefficients[18] * pDecodedSamples[-19]; + case 18: prediction += coefficients[17] * pDecodedSamples[-18]; + case 17: prediction += coefficients[16] * pDecodedSamples[-17]; + case 16: prediction += coefficients[15] * pDecodedSamples[-16]; + case 15: prediction += coefficients[14] * pDecodedSamples[-15]; + case 14: prediction += coefficients[13] * pDecodedSamples[-14]; + case 13: prediction += coefficients[12] * pDecodedSamples[-13]; + case 12: prediction += coefficients[11] * pDecodedSamples[-12]; + case 11: prediction += coefficients[10] * pDecodedSamples[-11]; + case 10: prediction += coefficients[ 9] * pDecodedSamples[-10]; + case 9: prediction += coefficients[ 8] * pDecodedSamples[- 9]; + case 8: prediction += coefficients[ 7] * pDecodedSamples[- 8]; + case 7: prediction += coefficients[ 6] * pDecodedSamples[- 7]; + case 6: prediction += coefficients[ 5] * pDecodedSamples[- 6]; + case 5: prediction += coefficients[ 4] * pDecodedSamples[- 5]; + case 4: prediction += coefficients[ 3] * pDecodedSamples[- 4]; + case 3: prediction += coefficients[ 2] * pDecodedSamples[- 3]; + case 2: prediction += coefficients[ 1] * pDecodedSamples[- 2]; + case 1: prediction += coefficients[ 0] * pDecodedSamples[- 1]; + } + + return (drflac_int32)(prediction >> shift); +} + +static DRFLAC_INLINE drflac_int32 drflac__calculate_prediction_64(drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) +{ + drflac_int64 prediction; + + DRFLAC_ASSERT(order <= 32); + + /* 64-bit version. */ + + /* This method is faster on the 32-bit build when compiling with VC++. See note below. */ +#ifndef DRFLAC_64BIT + if (order == 8) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; + } + else if (order == 7) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + } + else if (order == 3) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + } + else if (order == 6) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + } + else if (order == 5) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + } + else if (order == 4) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + } + else if (order == 12) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; + prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; + prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; + prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; + prediction += coefficients[11] * (drflac_int64)pDecodedSamples[-12]; + } + else if (order == 2) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + } + else if (order == 1) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + } + else if (order == 10) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; + prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; + prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; + } + else if (order == 9) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; + prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; + } + else if (order == 11) + { + prediction = coefficients[0] * (drflac_int64)pDecodedSamples[-1]; + prediction += coefficients[1] * (drflac_int64)pDecodedSamples[-2]; + prediction += coefficients[2] * (drflac_int64)pDecodedSamples[-3]; + prediction += coefficients[3] * (drflac_int64)pDecodedSamples[-4]; + prediction += coefficients[4] * (drflac_int64)pDecodedSamples[-5]; + prediction += coefficients[5] * (drflac_int64)pDecodedSamples[-6]; + prediction += coefficients[6] * (drflac_int64)pDecodedSamples[-7]; + prediction += coefficients[7] * (drflac_int64)pDecodedSamples[-8]; + prediction += coefficients[8] * (drflac_int64)pDecodedSamples[-9]; + prediction += coefficients[9] * (drflac_int64)pDecodedSamples[-10]; + prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; + } + else + { + int j; + + prediction = 0; + for (j = 0; j < (int)order; ++j) { + prediction += coefficients[j] * (drflac_int64)pDecodedSamples[-j-1]; + } + } +#endif + + /* + VC++ optimizes this to a single jmp instruction, but only the 64-bit build. The 32-bit build generates less efficient code for some + reason. The ugly version above is faster so we'll just switch between the two depending on the target platform. + */ +#ifdef DRFLAC_64BIT + prediction = 0; + switch (order) + { + case 32: prediction += coefficients[31] * (drflac_int64)pDecodedSamples[-32]; + case 31: prediction += coefficients[30] * (drflac_int64)pDecodedSamples[-31]; + case 30: prediction += coefficients[29] * (drflac_int64)pDecodedSamples[-30]; + case 29: prediction += coefficients[28] * (drflac_int64)pDecodedSamples[-29]; + case 28: prediction += coefficients[27] * (drflac_int64)pDecodedSamples[-28]; + case 27: prediction += coefficients[26] * (drflac_int64)pDecodedSamples[-27]; + case 26: prediction += coefficients[25] * (drflac_int64)pDecodedSamples[-26]; + case 25: prediction += coefficients[24] * (drflac_int64)pDecodedSamples[-25]; + case 24: prediction += coefficients[23] * (drflac_int64)pDecodedSamples[-24]; + case 23: prediction += coefficients[22] * (drflac_int64)pDecodedSamples[-23]; + case 22: prediction += coefficients[21] * (drflac_int64)pDecodedSamples[-22]; + case 21: prediction += coefficients[20] * (drflac_int64)pDecodedSamples[-21]; + case 20: prediction += coefficients[19] * (drflac_int64)pDecodedSamples[-20]; + case 19: prediction += coefficients[18] * (drflac_int64)pDecodedSamples[-19]; + case 18: prediction += coefficients[17] * (drflac_int64)pDecodedSamples[-18]; + case 17: prediction += coefficients[16] * (drflac_int64)pDecodedSamples[-17]; + case 16: prediction += coefficients[15] * (drflac_int64)pDecodedSamples[-16]; + case 15: prediction += coefficients[14] * (drflac_int64)pDecodedSamples[-15]; + case 14: prediction += coefficients[13] * (drflac_int64)pDecodedSamples[-14]; + case 13: prediction += coefficients[12] * (drflac_int64)pDecodedSamples[-13]; + case 12: prediction += coefficients[11] * (drflac_int64)pDecodedSamples[-12]; + case 11: prediction += coefficients[10] * (drflac_int64)pDecodedSamples[-11]; + case 10: prediction += coefficients[ 9] * (drflac_int64)pDecodedSamples[-10]; + case 9: prediction += coefficients[ 8] * (drflac_int64)pDecodedSamples[- 9]; + case 8: prediction += coefficients[ 7] * (drflac_int64)pDecodedSamples[- 8]; + case 7: prediction += coefficients[ 6] * (drflac_int64)pDecodedSamples[- 7]; + case 6: prediction += coefficients[ 5] * (drflac_int64)pDecodedSamples[- 6]; + case 5: prediction += coefficients[ 4] * (drflac_int64)pDecodedSamples[- 5]; + case 4: prediction += coefficients[ 3] * (drflac_int64)pDecodedSamples[- 4]; + case 3: prediction += coefficients[ 2] * (drflac_int64)pDecodedSamples[- 3]; + case 2: prediction += coefficients[ 1] * (drflac_int64)pDecodedSamples[- 2]; + case 1: prediction += coefficients[ 0] * (drflac_int64)pDecodedSamples[- 1]; + } +#endif + + return (drflac_int32)(prediction >> shift); +} + + +#if 0 +/* +Reference implementation for reading and decoding samples with residual. This is intentionally left unoptimized for the +sake of readability and should only be used as a reference. +*/ +static drflac_bool32 drflac__decode_samples_with_residual__rice__reference(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + drflac_uint32 i; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pSamplesOut != NULL); + + for (i = 0; i < count; ++i) { + drflac_uint32 zeroCounter = 0; + for (;;) { + drflac_uint8 bit; + if (!drflac__read_uint8(bs, 1, &bit)) { + return DRFLAC_FALSE; + } + + if (bit == 0) { + zeroCounter += 1; + } else { + break; + } + } + + drflac_uint32 decodedRice; + if (riceParam > 0) { + if (!drflac__read_uint32(bs, riceParam, &decodedRice)) { + return DRFLAC_FALSE; + } + } else { + decodedRice = 0; + } + + decodedRice |= (zeroCounter << riceParam); + if ((decodedRice & 0x01)) { + decodedRice = ~(decodedRice >> 1); + } else { + decodedRice = (decodedRice >> 1); + } + + + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + pSamplesOut[i] = decodedRice + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + } else { + pSamplesOut[i] = decodedRice + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + } + } + + return DRFLAC_TRUE; +} +#endif + +#if 0 +static drflac_bool32 drflac__read_rice_parts__reference(drflac_bs* bs, drflac_uint8 riceParam, drflac_uint32* pZeroCounterOut, drflac_uint32* pRiceParamPartOut) +{ + drflac_uint32 zeroCounter = 0; + drflac_uint32 decodedRice; + + for (;;) { + drflac_uint8 bit; + if (!drflac__read_uint8(bs, 1, &bit)) { + return DRFLAC_FALSE; + } + + if (bit == 0) { + zeroCounter += 1; + } else { + break; + } + } + + if (riceParam > 0) { + if (!drflac__read_uint32(bs, riceParam, &decodedRice)) { + return DRFLAC_FALSE; + } + } else { + decodedRice = 0; + } + + *pZeroCounterOut = zeroCounter; + *pRiceParamPartOut = decodedRice; + return DRFLAC_TRUE; +} +#endif + +#if 0 +static DRFLAC_INLINE drflac_bool32 drflac__read_rice_parts(drflac_bs* bs, drflac_uint8 riceParam, drflac_uint32* pZeroCounterOut, drflac_uint32* pRiceParamPartOut) +{ + drflac_cache_t riceParamMask; + drflac_uint32 zeroCounter; + drflac_uint32 setBitOffsetPlus1; + drflac_uint32 riceParamPart; + drflac_uint32 riceLength; + + DRFLAC_ASSERT(riceParam > 0); /* <-- riceParam should never be 0. drflac__read_rice_parts__param_equals_zero() should be used instead for this case. */ + + riceParamMask = DRFLAC_CACHE_L1_SELECTION_MASK(riceParam); + + zeroCounter = 0; + while (bs->cache == 0) { + zeroCounter += (drflac_uint32)DRFLAC_CACHE_L1_BITS_REMAINING(bs); + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + } + + setBitOffsetPlus1 = drflac__clz(bs->cache); + zeroCounter += setBitOffsetPlus1; + setBitOffsetPlus1 += 1; + + riceLength = setBitOffsetPlus1 + riceParam; + if (riceLength < DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + riceParamPart = (drflac_uint32)((bs->cache & (riceParamMask >> setBitOffsetPlus1)) >> DRFLAC_CACHE_L1_SELECTION_SHIFT(bs, riceLength)); + + bs->consumedBits += riceLength; + bs->cache <<= riceLength; + } else { + drflac_uint32 bitCountLo; + drflac_cache_t resultHi; + + bs->consumedBits += riceLength; + bs->cache <<= setBitOffsetPlus1 & (DRFLAC_CACHE_L1_SIZE_BITS(bs)-1); /* <-- Equivalent to "if (setBitOffsetPlus1 < DRFLAC_CACHE_L1_SIZE_BITS(bs)) { bs->cache <<= setBitOffsetPlus1; }" */ + + /* It straddles the cached data. It will never cover more than the next chunk. We just read the number in two parts and combine them. */ + bitCountLo = bs->consumedBits - DRFLAC_CACHE_L1_SIZE_BITS(bs); + resultHi = DRFLAC_CACHE_L1_SELECT_AND_SHIFT(bs, riceParam); /* <-- Use DRFLAC_CACHE_L1_SELECT_AND_SHIFT_SAFE() if ever this function allows riceParam=0. */ + + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { +#ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); +#endif + bs->cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); + bs->consumedBits = 0; +#ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs->cache; +#endif + } else { + /* Slow path. We need to fetch more data from the client. */ + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + if (bitCountLo > DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* This happens when we get to end of stream */ + return DRFLAC_FALSE; + } + } + + riceParamPart = (drflac_uint32)(resultHi | DRFLAC_CACHE_L1_SELECT_AND_SHIFT_SAFE(bs, bitCountLo)); + + bs->consumedBits += bitCountLo; + bs->cache <<= bitCountLo; + } + + pZeroCounterOut[0] = zeroCounter; + pRiceParamPartOut[0] = riceParamPart; + + return DRFLAC_TRUE; +} +#endif + +static DRFLAC_INLINE drflac_bool32 drflac__read_rice_parts_x1(drflac_bs* bs, drflac_uint8 riceParam, drflac_uint32* pZeroCounterOut, drflac_uint32* pRiceParamPartOut) +{ + drflac_uint32 riceParamPlus1 = riceParam + 1; + /*drflac_cache_t riceParamPlus1Mask = DRFLAC_CACHE_L1_SELECTION_MASK(riceParamPlus1);*/ + drflac_uint32 riceParamPlus1Shift = DRFLAC_CACHE_L1_SELECTION_SHIFT(bs, riceParamPlus1); + drflac_uint32 riceParamPlus1MaxConsumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs) - riceParamPlus1; + + /* + The idea here is to use local variables for the cache in an attempt to encourage the compiler to store them in registers. I have + no idea how this will work in practice... + */ + drflac_cache_t bs_cache = bs->cache; + drflac_uint32 bs_consumedBits = bs->consumedBits; + + /* The first thing to do is find the first unset bit. Most likely a bit will be set in the current cache line. */ + drflac_uint32 lzcount = drflac__clz(bs_cache); + if (lzcount < sizeof(bs_cache)*8) { + pZeroCounterOut[0] = lzcount; + + /* + It is most likely that the riceParam part (which comes after the zero counter) is also on this cache line. When extracting + this, we include the set bit from the unary coded part because it simplifies cache management. This bit will be handled + outside of this function at a higher level. + */ + extract_rice_param_part: + bs_cache <<= lzcount; + bs_consumedBits += lzcount; + + if (bs_consumedBits <= riceParamPlus1MaxConsumedBits) { + /* Getting here means the rice parameter part is wholly contained within the current cache line. */ + pRiceParamPartOut[0] = (drflac_uint32)(bs_cache >> riceParamPlus1Shift); + bs_cache <<= riceParamPlus1; + bs_consumedBits += riceParamPlus1; + } else { + drflac_uint32 riceParamPartHi; + drflac_uint32 riceParamPartLo; + drflac_uint32 riceParamPartLoBitCount; + + /* + Getting here means the rice parameter part straddles the cache line. We need to read from the tail of the current cache + line, reload the cache, and then combine it with the head of the next cache line. + */ + + /* Grab the high part of the rice parameter part. */ + riceParamPartHi = (drflac_uint32)(bs_cache >> riceParamPlus1Shift); + + /* Before reloading the cache we need to grab the size in bits of the low part. */ + riceParamPartLoBitCount = bs_consumedBits - riceParamPlus1MaxConsumedBits; + DRFLAC_ASSERT(riceParamPartLoBitCount > 0 && riceParamPartLoBitCount < 32); + + /* Now reload the cache. */ + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { + #ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); + #endif + bs_cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); + bs_consumedBits = riceParamPartLoBitCount; + #ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs_cache; + #endif + } else { + /* Slow path. We need to fetch more data from the client. */ + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + if (riceParamPartLoBitCount > DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* This happens when we get to end of stream */ + return DRFLAC_FALSE; + } + + bs_cache = bs->cache; + bs_consumedBits = bs->consumedBits + riceParamPartLoBitCount; + } + + /* We should now have enough information to construct the rice parameter part. */ + riceParamPartLo = (drflac_uint32)(bs_cache >> (DRFLAC_CACHE_L1_SELECTION_SHIFT(bs, riceParamPartLoBitCount))); + pRiceParamPartOut[0] = riceParamPartHi | riceParamPartLo; + + bs_cache <<= riceParamPartLoBitCount; + } + } else { + /* + Getting here means there are no bits set on the cache line. This is a less optimal case because we just wasted a call + to drflac__clz() and we need to reload the cache. + */ + drflac_uint32 zeroCounter = (drflac_uint32)(DRFLAC_CACHE_L1_SIZE_BITS(bs) - bs_consumedBits); + for (;;) { + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { + #ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); + #endif + bs_cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); + bs_consumedBits = 0; + #ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs_cache; + #endif + } else { + /* Slow path. We need to fetch more data from the client. */ + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + + bs_cache = bs->cache; + bs_consumedBits = bs->consumedBits; + } + + lzcount = drflac__clz(bs_cache); + zeroCounter += lzcount; + + if (lzcount < sizeof(bs_cache)*8) { + break; + } + } + + pZeroCounterOut[0] = zeroCounter; + goto extract_rice_param_part; + } + + /* Make sure the cache is restored at the end of it all. */ + bs->cache = bs_cache; + bs->consumedBits = bs_consumedBits; + + return DRFLAC_TRUE; +} + +static DRFLAC_INLINE drflac_bool32 drflac__seek_rice_parts(drflac_bs* bs, drflac_uint8 riceParam) +{ + drflac_uint32 riceParamPlus1 = riceParam + 1; + drflac_uint32 riceParamPlus1MaxConsumedBits = DRFLAC_CACHE_L1_SIZE_BITS(bs) - riceParamPlus1; + + /* + The idea here is to use local variables for the cache in an attempt to encourage the compiler to store them in registers. I have + no idea how this will work in practice... + */ + drflac_cache_t bs_cache = bs->cache; + drflac_uint32 bs_consumedBits = bs->consumedBits; + + /* The first thing to do is find the first unset bit. Most likely a bit will be set in the current cache line. */ + drflac_uint32 lzcount = drflac__clz(bs_cache); + if (lzcount < sizeof(bs_cache)*8) { + /* + It is most likely that the riceParam part (which comes after the zero counter) is also on this cache line. When extracting + this, we include the set bit from the unary coded part because it simplifies cache management. This bit will be handled + outside of this function at a higher level. + */ + extract_rice_param_part: + bs_cache <<= lzcount; + bs_consumedBits += lzcount; + + if (bs_consumedBits <= riceParamPlus1MaxConsumedBits) { + /* Getting here means the rice parameter part is wholly contained within the current cache line. */ + bs_cache <<= riceParamPlus1; + bs_consumedBits += riceParamPlus1; + } else { + /* + Getting here means the rice parameter part straddles the cache line. We need to read from the tail of the current cache + line, reload the cache, and then combine it with the head of the next cache line. + */ + + /* Before reloading the cache we need to grab the size in bits of the low part. */ + drflac_uint32 riceParamPartLoBitCount = bs_consumedBits - riceParamPlus1MaxConsumedBits; + DRFLAC_ASSERT(riceParamPartLoBitCount > 0 && riceParamPartLoBitCount < 32); + + /* Now reload the cache. */ + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { + #ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); + #endif + bs_cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); + bs_consumedBits = riceParamPartLoBitCount; + #ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs_cache; + #endif + } else { + /* Slow path. We need to fetch more data from the client. */ + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + + if (riceParamPartLoBitCount > DRFLAC_CACHE_L1_BITS_REMAINING(bs)) { + /* This happens when we get to end of stream */ + return DRFLAC_FALSE; + } + + bs_cache = bs->cache; + bs_consumedBits = bs->consumedBits + riceParamPartLoBitCount; + } + + bs_cache <<= riceParamPartLoBitCount; + } + } else { + /* + Getting here means there are no bits set on the cache line. This is a less optimal case because we just wasted a call + to drflac__clz() and we need to reload the cache. + */ + for (;;) { + if (bs->nextL2Line < DRFLAC_CACHE_L2_LINE_COUNT(bs)) { + #ifndef DR_FLAC_NO_CRC + drflac__update_crc16(bs); + #endif + bs_cache = drflac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]); + bs_consumedBits = 0; + #ifndef DR_FLAC_NO_CRC + bs->crc16Cache = bs_cache; + #endif + } else { + /* Slow path. We need to fetch more data from the client. */ + if (!drflac__reload_cache(bs)) { + return DRFLAC_FALSE; + } + + bs_cache = bs->cache; + bs_consumedBits = bs->consumedBits; + } + + lzcount = drflac__clz(bs_cache); + if (lzcount < sizeof(bs_cache)*8) { + break; + } + } + + goto extract_rice_param_part; + } + + /* Make sure the cache is restored at the end of it all. */ + bs->cache = bs_cache; + bs->consumedBits = bs_consumedBits; + + return DRFLAC_TRUE; +} + + +static drflac_bool32 drflac__decode_samples_with_residual__rice__scalar_zeroorder(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + drflac_uint32 zeroCountPart0; + drflac_uint32 riceParamPart0; + drflac_uint32 riceParamMask; + drflac_uint32 i; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pSamplesOut != NULL); + + (void)bitsPerSample; + (void)order; + (void)shift; + (void)coefficients; + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + + i = 0; + while (i < count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0)) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamPart0 &= riceParamMask; + riceParamPart0 |= (zeroCountPart0 << riceParam); + riceParamPart0 = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01]; + + pSamplesOut[i] = riceParamPart0; + + i += 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__scalar(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + drflac_uint32 zeroCountPart0 = 0; + drflac_uint32 zeroCountPart1 = 0; + drflac_uint32 zeroCountPart2 = 0; + drflac_uint32 zeroCountPart3 = 0; + drflac_uint32 riceParamPart0 = 0; + drflac_uint32 riceParamPart1 = 0; + drflac_uint32 riceParamPart2 = 0; + drflac_uint32 riceParamPart3 = 0; + drflac_uint32 riceParamMask; + const drflac_int32* pSamplesOutEnd; + drflac_uint32 i; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pSamplesOut != NULL); + + if (lpcOrder == 0) { + return drflac__decode_samples_with_residual__rice__scalar_zeroorder(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut); + } + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + pSamplesOutEnd = pSamplesOut + (count & ~3); + + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + while (pSamplesOut < pSamplesOutEnd) { + /* + Rice extraction. It's faster to do this one at a time against local variables than it is to use the x4 version + against an array. Not sure why, but perhaps it's making more efficient use of registers? + */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart1, &riceParamPart1) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart2, &riceParamPart2) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart3, &riceParamPart3)) { + return DRFLAC_FALSE; + } + + riceParamPart0 &= riceParamMask; + riceParamPart1 &= riceParamMask; + riceParamPart2 &= riceParamMask; + riceParamPart3 &= riceParamMask; + + riceParamPart0 |= (zeroCountPart0 << riceParam); + riceParamPart1 |= (zeroCountPart1 << riceParam); + riceParamPart2 |= (zeroCountPart2 << riceParam); + riceParamPart3 |= (zeroCountPart3 << riceParam); + + riceParamPart0 = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01]; + riceParamPart1 = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01]; + riceParamPart2 = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01]; + riceParamPart3 = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01]; + + pSamplesOut[0] = riceParamPart0 + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[1] = riceParamPart1 + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); + pSamplesOut[2] = riceParamPart2 + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); + pSamplesOut[3] = riceParamPart3 + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); + + pSamplesOut += 4; + } + } else { + while (pSamplesOut < pSamplesOutEnd) { + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart1, &riceParamPart1) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart2, &riceParamPart2) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart3, &riceParamPart3)) { + return DRFLAC_FALSE; + } + + riceParamPart0 &= riceParamMask; + riceParamPart1 &= riceParamMask; + riceParamPart2 &= riceParamMask; + riceParamPart3 &= riceParamMask; + + riceParamPart0 |= (zeroCountPart0 << riceParam); + riceParamPart1 |= (zeroCountPart1 << riceParam); + riceParamPart2 |= (zeroCountPart2 << riceParam); + riceParamPart3 |= (zeroCountPart3 << riceParam); + + riceParamPart0 = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01]; + riceParamPart1 = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01]; + riceParamPart2 = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01]; + riceParamPart3 = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01]; + + pSamplesOut[0] = riceParamPart0 + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[1] = riceParamPart1 + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); + pSamplesOut[2] = riceParamPart2 + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); + pSamplesOut[3] = riceParamPart3 + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); + + pSamplesOut += 4; + } + } + + i = (count & ~3); + while (i < count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0)) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamPart0 &= riceParamMask; + riceParamPart0 |= (zeroCountPart0 << riceParam); + riceParamPart0 = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01]; + /*riceParamPart0 = (riceParamPart0 >> 1) ^ (~(riceParamPart0 & 0x01) + 1);*/ + + /* Sample reconstruction. */ + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + pSamplesOut[0] = riceParamPart0 + drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + } else { + pSamplesOut[0] = riceParamPart0 + drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + } + + i += 1; + pSamplesOut += 1; + } + + return DRFLAC_TRUE; +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE __m128i drflac__mm_packs_interleaved_epi32(__m128i a, __m128i b) +{ + __m128i r; + + /* Pack. */ + r = _mm_packs_epi32(a, b); + + /* a3a2 a1a0 b3b2 b1b0 -> a3a2 b3b2 a1a0 b1b0 */ + r = _mm_shuffle_epi32(r, _MM_SHUFFLE(3, 1, 2, 0)); + + /* a3a2 b3b2 a1a0 b1b0 -> a3b3 a2b2 a1b1 a0b0 */ + r = _mm_shufflehi_epi16(r, _MM_SHUFFLE(3, 1, 2, 0)); + r = _mm_shufflelo_epi16(r, _MM_SHUFFLE(3, 1, 2, 0)); + + return r; +} +#endif + +#if defined(DRFLAC_SUPPORT_SSE41) +static DRFLAC_INLINE __m128i drflac__mm_not_si128(__m128i a) +{ + return _mm_xor_si128(a, _mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128())); +} + +static DRFLAC_INLINE __m128i drflac__mm_hadd_epi32(__m128i x) +{ + __m128i x64 = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2))); + __m128i x32 = _mm_shufflelo_epi16(x64, _MM_SHUFFLE(1, 0, 3, 2)); + return _mm_add_epi32(x64, x32); +} + +static DRFLAC_INLINE __m128i drflac__mm_hadd_epi64(__m128i x) +{ + return _mm_add_epi64(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2))); +} + +static DRFLAC_INLINE __m128i drflac__mm_srai_epi64(__m128i x, int count) +{ + /* + To simplify this we are assuming count < 32. This restriction allows us to work on a low side and a high side. The low side + is shifted with zero bits, whereas the right side is shifted with sign bits. + */ + __m128i lo = _mm_srli_epi64(x, count); + __m128i hi = _mm_srai_epi32(x, count); + + hi = _mm_and_si128(hi, _mm_set_epi32(0xFFFFFFFF, 0, 0xFFFFFFFF, 0)); /* The high part needs to have the low part cleared. */ + + return _mm_or_si128(lo, hi); +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__sse41_32(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + int i; + drflac_uint32 riceParamMask; + drflac_int32* pDecodedSamples = pSamplesOut; + drflac_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3); + drflac_uint32 zeroCountParts0 = 0; + drflac_uint32 zeroCountParts1 = 0; + drflac_uint32 zeroCountParts2 = 0; + drflac_uint32 zeroCountParts3 = 0; + drflac_uint32 riceParamParts0 = 0; + drflac_uint32 riceParamParts1 = 0; + drflac_uint32 riceParamParts2 = 0; + drflac_uint32 riceParamParts3 = 0; + __m128i coefficients128_0; + __m128i coefficients128_4; + __m128i coefficients128_8; + __m128i samples128_0; + __m128i samples128_4; + __m128i samples128_8; + __m128i riceParamMask128; + + const drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + riceParamMask128 = _mm_set1_epi32(riceParamMask); + + /* Pre-load. */ + coefficients128_0 = _mm_setzero_si128(); + coefficients128_4 = _mm_setzero_si128(); + coefficients128_8 = _mm_setzero_si128(); + + samples128_0 = _mm_setzero_si128(); + samples128_4 = _mm_setzero_si128(); + samples128_8 = _mm_setzero_si128(); + + /* + Pre-loading the coefficients and prior samples is annoying because we need to ensure we don't try reading more than + what's available in the input buffers. It would be convenient to use a fall-through switch to do this, but this results + in strict aliasing warnings with GCC. To work around this I'm just doing something hacky. This feels a bit convoluted + so I think there's opportunity for this to be simplified. + */ +#if 1 + { + int runningOrder = order; + + /* 0 - 3. */ + if (runningOrder >= 4) { + coefficients128_0 = _mm_loadu_si128((const __m128i*)(coefficients + 0)); + samples128_0 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 4)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_0 = _mm_set_epi32(0, coefficients[2], coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], pSamplesOut[-3], 0); break; + case 2: coefficients128_0 = _mm_set_epi32(0, 0, coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], 0, 0); break; + case 1: coefficients128_0 = _mm_set_epi32(0, 0, 0, coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* 4 - 7 */ + if (runningOrder >= 4) { + coefficients128_4 = _mm_loadu_si128((const __m128i*)(coefficients + 4)); + samples128_4 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 8)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_4 = _mm_set_epi32(0, coefficients[6], coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], pSamplesOut[-7], 0); break; + case 2: coefficients128_4 = _mm_set_epi32(0, 0, coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], 0, 0); break; + case 1: coefficients128_4 = _mm_set_epi32(0, 0, 0, coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* 8 - 11 */ + if (runningOrder == 4) { + coefficients128_8 = _mm_loadu_si128((const __m128i*)(coefficients + 8)); + samples128_8 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 12)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_8 = _mm_set_epi32(0, coefficients[10], coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], pSamplesOut[-11], 0); break; + case 2: coefficients128_8 = _mm_set_epi32(0, 0, coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], 0, 0); break; + case 1: coefficients128_8 = _mm_set_epi32(0, 0, 0, coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* Coefficients need to be shuffled for our streaming algorithm below to work. Samples are already in the correct order from the loading routine above. */ + coefficients128_0 = _mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(0, 1, 2, 3)); + coefficients128_4 = _mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(0, 1, 2, 3)); + coefficients128_8 = _mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(0, 1, 2, 3)); + } +#else + /* This causes strict-aliasing warnings with GCC. */ + switch (order) + { + case 12: ((drflac_int32*)&coefficients128_8)[0] = coefficients[11]; ((drflac_int32*)&samples128_8)[0] = pDecodedSamples[-12]; + case 11: ((drflac_int32*)&coefficients128_8)[1] = coefficients[10]; ((drflac_int32*)&samples128_8)[1] = pDecodedSamples[-11]; + case 10: ((drflac_int32*)&coefficients128_8)[2] = coefficients[ 9]; ((drflac_int32*)&samples128_8)[2] = pDecodedSamples[-10]; + case 9: ((drflac_int32*)&coefficients128_8)[3] = coefficients[ 8]; ((drflac_int32*)&samples128_8)[3] = pDecodedSamples[- 9]; + case 8: ((drflac_int32*)&coefficients128_4)[0] = coefficients[ 7]; ((drflac_int32*)&samples128_4)[0] = pDecodedSamples[- 8]; + case 7: ((drflac_int32*)&coefficients128_4)[1] = coefficients[ 6]; ((drflac_int32*)&samples128_4)[1] = pDecodedSamples[- 7]; + case 6: ((drflac_int32*)&coefficients128_4)[2] = coefficients[ 5]; ((drflac_int32*)&samples128_4)[2] = pDecodedSamples[- 6]; + case 5: ((drflac_int32*)&coefficients128_4)[3] = coefficients[ 4]; ((drflac_int32*)&samples128_4)[3] = pDecodedSamples[- 5]; + case 4: ((drflac_int32*)&coefficients128_0)[0] = coefficients[ 3]; ((drflac_int32*)&samples128_0)[0] = pDecodedSamples[- 4]; + case 3: ((drflac_int32*)&coefficients128_0)[1] = coefficients[ 2]; ((drflac_int32*)&samples128_0)[1] = pDecodedSamples[- 3]; + case 2: ((drflac_int32*)&coefficients128_0)[2] = coefficients[ 1]; ((drflac_int32*)&samples128_0)[2] = pDecodedSamples[- 2]; + case 1: ((drflac_int32*)&coefficients128_0)[3] = coefficients[ 0]; ((drflac_int32*)&samples128_0)[3] = pDecodedSamples[- 1]; + } +#endif + + /* For this version we are doing one sample at a time. */ + while (pDecodedSamples < pDecodedSamplesEnd) { + __m128i prediction128; + __m128i zeroCountPart128; + __m128i riceParamPart128; + + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts1, &riceParamParts1) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts2, &riceParamParts2) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts3, &riceParamParts3)) { + return DRFLAC_FALSE; + } + + zeroCountPart128 = _mm_set_epi32(zeroCountParts3, zeroCountParts2, zeroCountParts1, zeroCountParts0); + riceParamPart128 = _mm_set_epi32(riceParamParts3, riceParamParts2, riceParamParts1, riceParamParts0); + + riceParamPart128 = _mm_and_si128(riceParamPart128, riceParamMask128); + riceParamPart128 = _mm_or_si128(riceParamPart128, _mm_slli_epi32(zeroCountPart128, riceParam)); + riceParamPart128 = _mm_xor_si128(_mm_srli_epi32(riceParamPart128, 1), _mm_add_epi32(drflac__mm_not_si128(_mm_and_si128(riceParamPart128, _mm_set1_epi32(0x01))), _mm_set1_epi32(0x01))); /* <-- SSE2 compatible */ + /*riceParamPart128 = _mm_xor_si128(_mm_srli_epi32(riceParamPart128, 1), _mm_mullo_epi32(_mm_and_si128(riceParamPart128, _mm_set1_epi32(0x01)), _mm_set1_epi32(0xFFFFFFFF)));*/ /* <-- Only supported from SSE4.1 and is slower in my testing... */ + + if (order <= 4) { + for (i = 0; i < 4; i += 1) { + prediction128 = _mm_mullo_epi32(coefficients128_0, samples128_0); + + /* Horizontal add and shift. */ + prediction128 = drflac__mm_hadd_epi32(prediction128); + prediction128 = _mm_srai_epi32(prediction128, shift); + prediction128 = _mm_add_epi32(riceParamPart128, prediction128); + + samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4); + riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4); + } + } else if (order <= 8) { + for (i = 0; i < 4; i += 1) { + prediction128 = _mm_mullo_epi32(coefficients128_4, samples128_4); + prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_0, samples128_0)); + + /* Horizontal add and shift. */ + prediction128 = drflac__mm_hadd_epi32(prediction128); + prediction128 = _mm_srai_epi32(prediction128, shift); + prediction128 = _mm_add_epi32(riceParamPart128, prediction128); + + samples128_4 = _mm_alignr_epi8(samples128_0, samples128_4, 4); + samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4); + riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4); + } + } else { + for (i = 0; i < 4; i += 1) { + prediction128 = _mm_mullo_epi32(coefficients128_8, samples128_8); + prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_4, samples128_4)); + prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_0, samples128_0)); + + /* Horizontal add and shift. */ + prediction128 = drflac__mm_hadd_epi32(prediction128); + prediction128 = _mm_srai_epi32(prediction128, shift); + prediction128 = _mm_add_epi32(riceParamPart128, prediction128); + + samples128_8 = _mm_alignr_epi8(samples128_4, samples128_8, 4); + samples128_4 = _mm_alignr_epi8(samples128_0, samples128_4, 4); + samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4); + riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4); + } + } + + /* We store samples in groups of 4. */ + _mm_storeu_si128((__m128i*)pDecodedSamples, samples128_0); + pDecodedSamples += 4; + } + + /* Make sure we process the last few samples. */ + i = (count & ~3); + while (i < (int)count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0)) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamParts0 &= riceParamMask; + riceParamParts0 |= (zeroCountParts0 << riceParam); + riceParamParts0 = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01]; + + /* Sample reconstruction. */ + pDecodedSamples[0] = riceParamParts0 + drflac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples); + + i += 1; + pDecodedSamples += 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__sse41_64(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + int i; + drflac_uint32 riceParamMask; + drflac_int32* pDecodedSamples = pSamplesOut; + drflac_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3); + drflac_uint32 zeroCountParts0 = 0; + drflac_uint32 zeroCountParts1 = 0; + drflac_uint32 zeroCountParts2 = 0; + drflac_uint32 zeroCountParts3 = 0; + drflac_uint32 riceParamParts0 = 0; + drflac_uint32 riceParamParts1 = 0; + drflac_uint32 riceParamParts2 = 0; + drflac_uint32 riceParamParts3 = 0; + __m128i coefficients128_0; + __m128i coefficients128_4; + __m128i coefficients128_8; + __m128i samples128_0; + __m128i samples128_4; + __m128i samples128_8; + __m128i prediction128; + __m128i riceParamMask128; + + const drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + + DRFLAC_ASSERT(order <= 12); + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + riceParamMask128 = _mm_set1_epi32(riceParamMask); + + prediction128 = _mm_setzero_si128(); + + /* Pre-load. */ + coefficients128_0 = _mm_setzero_si128(); + coefficients128_4 = _mm_setzero_si128(); + coefficients128_8 = _mm_setzero_si128(); + + samples128_0 = _mm_setzero_si128(); + samples128_4 = _mm_setzero_si128(); + samples128_8 = _mm_setzero_si128(); + +#if 1 + { + int runningOrder = order; + + /* 0 - 3. */ + if (runningOrder >= 4) { + coefficients128_0 = _mm_loadu_si128((const __m128i*)(coefficients + 0)); + samples128_0 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 4)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_0 = _mm_set_epi32(0, coefficients[2], coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], pSamplesOut[-3], 0); break; + case 2: coefficients128_0 = _mm_set_epi32(0, 0, coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], 0, 0); break; + case 1: coefficients128_0 = _mm_set_epi32(0, 0, 0, coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* 4 - 7 */ + if (runningOrder >= 4) { + coefficients128_4 = _mm_loadu_si128((const __m128i*)(coefficients + 4)); + samples128_4 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 8)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_4 = _mm_set_epi32(0, coefficients[6], coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], pSamplesOut[-7], 0); break; + case 2: coefficients128_4 = _mm_set_epi32(0, 0, coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], 0, 0); break; + case 1: coefficients128_4 = _mm_set_epi32(0, 0, 0, coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* 8 - 11 */ + if (runningOrder == 4) { + coefficients128_8 = _mm_loadu_si128((const __m128i*)(coefficients + 8)); + samples128_8 = _mm_loadu_si128((const __m128i*)(pSamplesOut - 12)); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: coefficients128_8 = _mm_set_epi32(0, coefficients[10], coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], pSamplesOut[-11], 0); break; + case 2: coefficients128_8 = _mm_set_epi32(0, 0, coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], 0, 0); break; + case 1: coefficients128_8 = _mm_set_epi32(0, 0, 0, coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], 0, 0, 0); break; + } + runningOrder = 0; + } + + /* Coefficients need to be shuffled for our streaming algorithm below to work. Samples are already in the correct order from the loading routine above. */ + coefficients128_0 = _mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(0, 1, 2, 3)); + coefficients128_4 = _mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(0, 1, 2, 3)); + coefficients128_8 = _mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(0, 1, 2, 3)); + } +#else + switch (order) + { + case 12: ((drflac_int32*)&coefficients128_8)[0] = coefficients[11]; ((drflac_int32*)&samples128_8)[0] = pDecodedSamples[-12]; + case 11: ((drflac_int32*)&coefficients128_8)[1] = coefficients[10]; ((drflac_int32*)&samples128_8)[1] = pDecodedSamples[-11]; + case 10: ((drflac_int32*)&coefficients128_8)[2] = coefficients[ 9]; ((drflac_int32*)&samples128_8)[2] = pDecodedSamples[-10]; + case 9: ((drflac_int32*)&coefficients128_8)[3] = coefficients[ 8]; ((drflac_int32*)&samples128_8)[3] = pDecodedSamples[- 9]; + case 8: ((drflac_int32*)&coefficients128_4)[0] = coefficients[ 7]; ((drflac_int32*)&samples128_4)[0] = pDecodedSamples[- 8]; + case 7: ((drflac_int32*)&coefficients128_4)[1] = coefficients[ 6]; ((drflac_int32*)&samples128_4)[1] = pDecodedSamples[- 7]; + case 6: ((drflac_int32*)&coefficients128_4)[2] = coefficients[ 5]; ((drflac_int32*)&samples128_4)[2] = pDecodedSamples[- 6]; + case 5: ((drflac_int32*)&coefficients128_4)[3] = coefficients[ 4]; ((drflac_int32*)&samples128_4)[3] = pDecodedSamples[- 5]; + case 4: ((drflac_int32*)&coefficients128_0)[0] = coefficients[ 3]; ((drflac_int32*)&samples128_0)[0] = pDecodedSamples[- 4]; + case 3: ((drflac_int32*)&coefficients128_0)[1] = coefficients[ 2]; ((drflac_int32*)&samples128_0)[1] = pDecodedSamples[- 3]; + case 2: ((drflac_int32*)&coefficients128_0)[2] = coefficients[ 1]; ((drflac_int32*)&samples128_0)[2] = pDecodedSamples[- 2]; + case 1: ((drflac_int32*)&coefficients128_0)[3] = coefficients[ 0]; ((drflac_int32*)&samples128_0)[3] = pDecodedSamples[- 1]; + } +#endif + + /* For this version we are doing one sample at a time. */ + while (pDecodedSamples < pDecodedSamplesEnd) { + __m128i zeroCountPart128; + __m128i riceParamPart128; + + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts1, &riceParamParts1) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts2, &riceParamParts2) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts3, &riceParamParts3)) { + return DRFLAC_FALSE; + } + + zeroCountPart128 = _mm_set_epi32(zeroCountParts3, zeroCountParts2, zeroCountParts1, zeroCountParts0); + riceParamPart128 = _mm_set_epi32(riceParamParts3, riceParamParts2, riceParamParts1, riceParamParts0); + + riceParamPart128 = _mm_and_si128(riceParamPart128, riceParamMask128); + riceParamPart128 = _mm_or_si128(riceParamPart128, _mm_slli_epi32(zeroCountPart128, riceParam)); + riceParamPart128 = _mm_xor_si128(_mm_srli_epi32(riceParamPart128, 1), _mm_add_epi32(drflac__mm_not_si128(_mm_and_si128(riceParamPart128, _mm_set1_epi32(1))), _mm_set1_epi32(1))); + + for (i = 0; i < 4; i += 1) { + prediction128 = _mm_xor_si128(prediction128, prediction128); /* Reset to 0. */ + + switch (order) + { + case 12: + case 11: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_8, _MM_SHUFFLE(1, 1, 0, 0)))); + case 10: + case 9: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_8, _MM_SHUFFLE(3, 3, 2, 2)))); + case 8: + case 7: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_4, _MM_SHUFFLE(1, 1, 0, 0)))); + case 6: + case 5: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_4, _MM_SHUFFLE(3, 3, 2, 2)))); + case 4: + case 3: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_0, _MM_SHUFFLE(1, 1, 0, 0)))); + case 2: + case 1: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_0, _MM_SHUFFLE(3, 3, 2, 2)))); + } + + /* Horizontal add and shift. */ + prediction128 = drflac__mm_hadd_epi64(prediction128); + prediction128 = drflac__mm_srai_epi64(prediction128, shift); + prediction128 = _mm_add_epi32(riceParamPart128, prediction128); + + /* Our value should be sitting in prediction128[0]. We need to combine this with our SSE samples. */ + samples128_8 = _mm_alignr_epi8(samples128_4, samples128_8, 4); + samples128_4 = _mm_alignr_epi8(samples128_0, samples128_4, 4); + samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4); + + /* Slide our rice parameter down so that the value in position 0 contains the next one to process. */ + riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4); + } + + /* We store samples in groups of 4. */ + _mm_storeu_si128((__m128i*)pDecodedSamples, samples128_0); + pDecodedSamples += 4; + } + + /* Make sure we process the last few samples. */ + i = (count & ~3); + while (i < (int)count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0)) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamParts0 &= riceParamMask; + riceParamParts0 |= (zeroCountParts0 << riceParam); + riceParamParts0 = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01]; + + /* Sample reconstruction. */ + pDecodedSamples[0] = riceParamParts0 + drflac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples); + + i += 1; + pDecodedSamples += 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__sse41(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pSamplesOut != NULL); + + /* In my testing the order is rarely > 12, so in this case I'm going to simplify the SSE implementation by only handling order <= 12. */ + if (lpcOrder > 0 && lpcOrder <= 12) { + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + return drflac__decode_samples_with_residual__rice__sse41_64(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut); + } else { + return drflac__decode_samples_with_residual__rice__sse41_32(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut); + } + } else { + return drflac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac__vst2q_s32(drflac_int32* p, int32x4x2_t x) +{ + vst1q_s32(p+0, x.val[0]); + vst1q_s32(p+4, x.val[1]); +} + +static DRFLAC_INLINE void drflac__vst2q_u32(drflac_uint32* p, uint32x4x2_t x) +{ + vst1q_u32(p+0, x.val[0]); + vst1q_u32(p+4, x.val[1]); +} + +static DRFLAC_INLINE void drflac__vst2q_f32(float* p, float32x4x2_t x) +{ + vst1q_f32(p+0, x.val[0]); + vst1q_f32(p+4, x.val[1]); +} + +static DRFLAC_INLINE void drflac__vst2q_s16(drflac_int16* p, int16x4x2_t x) +{ + vst1q_s16(p, vcombine_s16(x.val[0], x.val[1])); +} + +static DRFLAC_INLINE void drflac__vst2q_u16(drflac_uint16* p, uint16x4x2_t x) +{ + vst1q_u16(p, vcombine_u16(x.val[0], x.val[1])); +} + +static DRFLAC_INLINE int32x4_t drflac__vdupq_n_s32x4(drflac_int32 x3, drflac_int32 x2, drflac_int32 x1, drflac_int32 x0) +{ + drflac_int32 x[4]; + x[3] = x3; + x[2] = x2; + x[1] = x1; + x[0] = x0; + return vld1q_s32(x); +} + +static DRFLAC_INLINE int32x4_t drflac__valignrq_s32_1(int32x4_t a, int32x4_t b) +{ + /* Equivalent to SSE's _mm_alignr_epi8(a, b, 4) */ + + /* Reference */ + /*return drflac__vdupq_n_s32x4( + vgetq_lane_s32(a, 0), + vgetq_lane_s32(b, 3), + vgetq_lane_s32(b, 2), + vgetq_lane_s32(b, 1) + );*/ + + return vextq_s32(b, a, 1); +} + +static DRFLAC_INLINE uint32x4_t drflac__valignrq_u32_1(uint32x4_t a, uint32x4_t b) +{ + /* Equivalent to SSE's _mm_alignr_epi8(a, b, 4) */ + + /* Reference */ + /*return drflac__vdupq_n_s32x4( + vgetq_lane_s32(a, 0), + vgetq_lane_s32(b, 3), + vgetq_lane_s32(b, 2), + vgetq_lane_s32(b, 1) + );*/ + + return vextq_u32(b, a, 1); +} + +static DRFLAC_INLINE int32x2_t drflac__vhaddq_s32(int32x4_t x) +{ + /* The sum must end up in position 0. */ + + /* Reference */ + /*return vdupq_n_s32( + vgetq_lane_s32(x, 3) + + vgetq_lane_s32(x, 2) + + vgetq_lane_s32(x, 1) + + vgetq_lane_s32(x, 0) + );*/ + + int32x2_t r = vadd_s32(vget_high_s32(x), vget_low_s32(x)); + return vpadd_s32(r, r); +} + +static DRFLAC_INLINE int64x1_t drflac__vhaddq_s64(int64x2_t x) +{ + return vadd_s64(vget_high_s64(x), vget_low_s64(x)); +} + +static DRFLAC_INLINE int32x4_t drflac__vrevq_s32(int32x4_t x) +{ + /* Reference */ + /*return drflac__vdupq_n_s32x4( + vgetq_lane_s32(x, 0), + vgetq_lane_s32(x, 1), + vgetq_lane_s32(x, 2), + vgetq_lane_s32(x, 3) + );*/ + + return vrev64q_s32(vcombine_s32(vget_high_s32(x), vget_low_s32(x))); +} + +static DRFLAC_INLINE int32x4_t drflac__vnotq_s32(int32x4_t x) +{ + return veorq_s32(x, vdupq_n_s32(0xFFFFFFFF)); +} + +static DRFLAC_INLINE uint32x4_t drflac__vnotq_u32(uint32x4_t x) +{ + return veorq_u32(x, vdupq_n_u32(0xFFFFFFFF)); +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__neon_32(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + int i; + drflac_uint32 riceParamMask; + drflac_int32* pDecodedSamples = pSamplesOut; + drflac_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3); + drflac_uint32 zeroCountParts[4]; + drflac_uint32 riceParamParts[4]; + int32x4_t coefficients128_0; + int32x4_t coefficients128_4; + int32x4_t coefficients128_8; + int32x4_t samples128_0; + int32x4_t samples128_4; + int32x4_t samples128_8; + uint32x4_t riceParamMask128; + int32x4_t riceParam128; + int32x2_t shift64; + uint32x4_t one128; + + const drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + riceParamMask128 = vdupq_n_u32(riceParamMask); + + riceParam128 = vdupq_n_s32(riceParam); + shift64 = vdup_n_s32(-shift); /* Negate the shift because we'll be doing a variable shift using vshlq_s32(). */ + one128 = vdupq_n_u32(1); + + /* + Pre-loading the coefficients and prior samples is annoying because we need to ensure we don't try reading more than + what's available in the input buffers. It would be conenient to use a fall-through switch to do this, but this results + in strict aliasing warnings with GCC. To work around this I'm just doing something hacky. This feels a bit convoluted + so I think there's opportunity for this to be simplified. + */ + { + int runningOrder = order; + drflac_int32 tempC[4] = {0, 0, 0, 0}; + drflac_int32 tempS[4] = {0, 0, 0, 0}; + + /* 0 - 3. */ + if (runningOrder >= 4) { + coefficients128_0 = vld1q_s32(coefficients + 0); + samples128_0 = vld1q_s32(pSamplesOut - 4); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[2]; tempS[1] = pSamplesOut[-3]; /* fallthrough */ + case 2: tempC[1] = coefficients[1]; tempS[2] = pSamplesOut[-2]; /* fallthrough */ + case 1: tempC[0] = coefficients[0]; tempS[3] = pSamplesOut[-1]; /* fallthrough */ + } + + coefficients128_0 = vld1q_s32(tempC); + samples128_0 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* 4 - 7 */ + if (runningOrder >= 4) { + coefficients128_4 = vld1q_s32(coefficients + 4); + samples128_4 = vld1q_s32(pSamplesOut - 8); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[6]; tempS[1] = pSamplesOut[-7]; /* fallthrough */ + case 2: tempC[1] = coefficients[5]; tempS[2] = pSamplesOut[-6]; /* fallthrough */ + case 1: tempC[0] = coefficients[4]; tempS[3] = pSamplesOut[-5]; /* fallthrough */ + } + + coefficients128_4 = vld1q_s32(tempC); + samples128_4 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* 8 - 11 */ + if (runningOrder == 4) { + coefficients128_8 = vld1q_s32(coefficients + 8); + samples128_8 = vld1q_s32(pSamplesOut - 12); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[10]; tempS[1] = pSamplesOut[-11]; /* fallthrough */ + case 2: tempC[1] = coefficients[ 9]; tempS[2] = pSamplesOut[-10]; /* fallthrough */ + case 1: tempC[0] = coefficients[ 8]; tempS[3] = pSamplesOut[- 9]; /* fallthrough */ + } + + coefficients128_8 = vld1q_s32(tempC); + samples128_8 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* Coefficients need to be shuffled for our streaming algorithm below to work. Samples are already in the correct order from the loading routine above. */ + coefficients128_0 = drflac__vrevq_s32(coefficients128_0); + coefficients128_4 = drflac__vrevq_s32(coefficients128_4); + coefficients128_8 = drflac__vrevq_s32(coefficients128_8); + } + + /* For this version we are doing one sample at a time. */ + while (pDecodedSamples < pDecodedSamplesEnd) { + int32x4_t prediction128; + int32x2_t prediction64; + uint32x4_t zeroCountPart128; + uint32x4_t riceParamPart128; + + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[1], &riceParamParts[1]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[2], &riceParamParts[2]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[3], &riceParamParts[3])) { + return DRFLAC_FALSE; + } + + zeroCountPart128 = vld1q_u32(zeroCountParts); + riceParamPart128 = vld1q_u32(riceParamParts); + + riceParamPart128 = vandq_u32(riceParamPart128, riceParamMask128); + riceParamPart128 = vorrq_u32(riceParamPart128, vshlq_u32(zeroCountPart128, riceParam128)); + riceParamPart128 = veorq_u32(vshrq_n_u32(riceParamPart128, 1), vaddq_u32(drflac__vnotq_u32(vandq_u32(riceParamPart128, one128)), one128)); + + if (order <= 4) { + for (i = 0; i < 4; i += 1) { + prediction128 = vmulq_s32(coefficients128_0, samples128_0); + + /* Horizontal add and shift. */ + prediction64 = drflac__vhaddq_s32(prediction128); + prediction64 = vshl_s32(prediction64, shift64); + prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128))); + + samples128_0 = drflac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0); + riceParamPart128 = drflac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128); + } + } else if (order <= 8) { + for (i = 0; i < 4; i += 1) { + prediction128 = vmulq_s32(coefficients128_4, samples128_4); + prediction128 = vmlaq_s32(prediction128, coefficients128_0, samples128_0); + + /* Horizontal add and shift. */ + prediction64 = drflac__vhaddq_s32(prediction128); + prediction64 = vshl_s32(prediction64, shift64); + prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128))); + + samples128_4 = drflac__valignrq_s32_1(samples128_0, samples128_4); + samples128_0 = drflac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0); + riceParamPart128 = drflac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128); + } + } else { + for (i = 0; i < 4; i += 1) { + prediction128 = vmulq_s32(coefficients128_8, samples128_8); + prediction128 = vmlaq_s32(prediction128, coefficients128_4, samples128_4); + prediction128 = vmlaq_s32(prediction128, coefficients128_0, samples128_0); + + /* Horizontal add and shift. */ + prediction64 = drflac__vhaddq_s32(prediction128); + prediction64 = vshl_s32(prediction64, shift64); + prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128))); + + samples128_8 = drflac__valignrq_s32_1(samples128_4, samples128_8); + samples128_4 = drflac__valignrq_s32_1(samples128_0, samples128_4); + samples128_0 = drflac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0); + riceParamPart128 = drflac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128); + } + } + + /* We store samples in groups of 4. */ + vst1q_s32(pDecodedSamples, samples128_0); + pDecodedSamples += 4; + } + + /* Make sure we process the last few samples. */ + i = (count & ~3); + while (i < (int)count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0])) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamParts[0] &= riceParamMask; + riceParamParts[0] |= (zeroCountParts[0] << riceParam); + riceParamParts[0] = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01]; + + /* Sample reconstruction. */ + pDecodedSamples[0] = riceParamParts[0] + drflac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples); + + i += 1; + pDecodedSamples += 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__neon_64(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 order, drflac_int32 shift, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + int i; + drflac_uint32 riceParamMask; + drflac_int32* pDecodedSamples = pSamplesOut; + drflac_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3); + drflac_uint32 zeroCountParts[4]; + drflac_uint32 riceParamParts[4]; + int32x4_t coefficients128_0; + int32x4_t coefficients128_4; + int32x4_t coefficients128_8; + int32x4_t samples128_0; + int32x4_t samples128_4; + int32x4_t samples128_8; + uint32x4_t riceParamMask128; + int32x4_t riceParam128; + int64x1_t shift64; + uint32x4_t one128; + int64x2_t prediction128 = { 0 }; + uint32x4_t zeroCountPart128; + uint32x4_t riceParamPart128; + + const drflac_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; + + riceParamMask = (drflac_uint32)~((~0UL) << riceParam); + riceParamMask128 = vdupq_n_u32(riceParamMask); + + riceParam128 = vdupq_n_s32(riceParam); + shift64 = vdup_n_s64(-shift); /* Negate the shift because we'll be doing a variable shift using vshlq_s32(). */ + one128 = vdupq_n_u32(1); + + /* + Pre-loading the coefficients and prior samples is annoying because we need to ensure we don't try reading more than + what's available in the input buffers. It would be convenient to use a fall-through switch to do this, but this results + in strict aliasing warnings with GCC. To work around this I'm just doing something hacky. This feels a bit convoluted + so I think there's opportunity for this to be simplified. + */ + { + int runningOrder = order; + drflac_int32 tempC[4] = {0, 0, 0, 0}; + drflac_int32 tempS[4] = {0, 0, 0, 0}; + + /* 0 - 3. */ + if (runningOrder >= 4) { + coefficients128_0 = vld1q_s32(coefficients + 0); + samples128_0 = vld1q_s32(pSamplesOut - 4); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[2]; tempS[1] = pSamplesOut[-3]; /* fallthrough */ + case 2: tempC[1] = coefficients[1]; tempS[2] = pSamplesOut[-2]; /* fallthrough */ + case 1: tempC[0] = coefficients[0]; tempS[3] = pSamplesOut[-1]; /* fallthrough */ + } + + coefficients128_0 = vld1q_s32(tempC); + samples128_0 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* 4 - 7 */ + if (runningOrder >= 4) { + coefficients128_4 = vld1q_s32(coefficients + 4); + samples128_4 = vld1q_s32(pSamplesOut - 8); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[6]; tempS[1] = pSamplesOut[-7]; /* fallthrough */ + case 2: tempC[1] = coefficients[5]; tempS[2] = pSamplesOut[-6]; /* fallthrough */ + case 1: tempC[0] = coefficients[4]; tempS[3] = pSamplesOut[-5]; /* fallthrough */ + } + + coefficients128_4 = vld1q_s32(tempC); + samples128_4 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* 8 - 11 */ + if (runningOrder == 4) { + coefficients128_8 = vld1q_s32(coefficients + 8); + samples128_8 = vld1q_s32(pSamplesOut - 12); + runningOrder -= 4; + } else { + switch (runningOrder) { + case 3: tempC[2] = coefficients[10]; tempS[1] = pSamplesOut[-11]; /* fallthrough */ + case 2: tempC[1] = coefficients[ 9]; tempS[2] = pSamplesOut[-10]; /* fallthrough */ + case 1: tempC[0] = coefficients[ 8]; tempS[3] = pSamplesOut[- 9]; /* fallthrough */ + } + + coefficients128_8 = vld1q_s32(tempC); + samples128_8 = vld1q_s32(tempS); + runningOrder = 0; + } + + /* Coefficients need to be shuffled for our streaming algorithm below to work. Samples are already in the correct order from the loading routine above. */ + coefficients128_0 = drflac__vrevq_s32(coefficients128_0); + coefficients128_4 = drflac__vrevq_s32(coefficients128_4); + coefficients128_8 = drflac__vrevq_s32(coefficients128_8); + } + + /* For this version we are doing one sample at a time. */ + while (pDecodedSamples < pDecodedSamplesEnd) { + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[1], &riceParamParts[1]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[2], &riceParamParts[2]) || + !drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[3], &riceParamParts[3])) { + return DRFLAC_FALSE; + } + + zeroCountPart128 = vld1q_u32(zeroCountParts); + riceParamPart128 = vld1q_u32(riceParamParts); + + riceParamPart128 = vandq_u32(riceParamPart128, riceParamMask128); + riceParamPart128 = vorrq_u32(riceParamPart128, vshlq_u32(zeroCountPart128, riceParam128)); + riceParamPart128 = veorq_u32(vshrq_n_u32(riceParamPart128, 1), vaddq_u32(drflac__vnotq_u32(vandq_u32(riceParamPart128, one128)), one128)); + + for (i = 0; i < 4; i += 1) { + int64x1_t prediction64; + + prediction128 = veorq_s64(prediction128, prediction128); /* Reset to 0. */ + switch (order) + { + case 12: + case 11: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_8), vget_low_s32(samples128_8))); + case 10: + case 9: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_8), vget_high_s32(samples128_8))); + case 8: + case 7: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_4), vget_low_s32(samples128_4))); + case 6: + case 5: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_4), vget_high_s32(samples128_4))); + case 4: + case 3: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_0), vget_low_s32(samples128_0))); + case 2: + case 1: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_0), vget_high_s32(samples128_0))); + } + + /* Horizontal add and shift. */ + prediction64 = drflac__vhaddq_s64(prediction128); + prediction64 = vshl_s64(prediction64, shift64); + prediction64 = vadd_s64(prediction64, vdup_n_s64(vgetq_lane_u32(riceParamPart128, 0))); + + /* Our value should be sitting in prediction64[0]. We need to combine this with our SSE samples. */ + samples128_8 = drflac__valignrq_s32_1(samples128_4, samples128_8); + samples128_4 = drflac__valignrq_s32_1(samples128_0, samples128_4); + samples128_0 = drflac__valignrq_s32_1(vcombine_s32(vreinterpret_s32_s64(prediction64), vdup_n_s32(0)), samples128_0); + + /* Slide our rice parameter down so that the value in position 0 contains the next one to process. */ + riceParamPart128 = drflac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128); + } + + /* We store samples in groups of 4. */ + vst1q_s32(pDecodedSamples, samples128_0); + pDecodedSamples += 4; + } + + /* Make sure we process the last few samples. */ + i = (count & ~3); + while (i < (int)count) { + /* Rice extraction. */ + if (!drflac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0])) { + return DRFLAC_FALSE; + } + + /* Rice reconstruction. */ + riceParamParts[0] &= riceParamMask; + riceParamParts[0] |= (zeroCountParts[0] << riceParam); + riceParamParts[0] = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01]; + + /* Sample reconstruction. */ + pDecodedSamples[0] = riceParamParts[0] + drflac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples); + + i += 1; + pDecodedSamples += 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples_with_residual__rice__neon(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(pSamplesOut != NULL); + + /* In my testing the order is rarely > 12, so in this case I'm going to simplify the NEON implementation by only handling order <= 12. */ + if (lpcOrder > 0 && lpcOrder <= 12) { + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + return drflac__decode_samples_with_residual__rice__neon_64(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut); + } else { + return drflac__decode_samples_with_residual__rice__neon_32(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut); + } + } else { + return drflac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + } +} +#endif + +static drflac_bool32 drflac__decode_samples_with_residual__rice(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 riceParam, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ +#if defined(DRFLAC_SUPPORT_SSE41) + if (drflac__gIsSSE41Supported) { + return drflac__decode_samples_with_residual__rice__sse41(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported) { + return drflac__decode_samples_with_residual__rice__neon(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + } else +#endif + { + /* Scalar fallback. */ + #if 0 + return drflac__decode_samples_with_residual__rice__reference(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + #else + return drflac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut); + #endif + } +} + +/* Reads and seeks past a string of residual values as Rice codes. The decoder should be sitting on the first bit of the Rice codes. */ +static drflac_bool32 drflac__read_and_seek_residual__rice(drflac_bs* bs, drflac_uint32 count, drflac_uint8 riceParam) +{ + drflac_uint32 i; + + DRFLAC_ASSERT(bs != NULL); + + for (i = 0; i < count; ++i) { + if (!drflac__seek_rice_parts(bs, riceParam)) { + return DRFLAC_FALSE; + } + } + + return DRFLAC_TRUE; +} + +#if defined(__clang__) +__attribute__((no_sanitize("signed-integer-overflow"))) +#endif +static drflac_bool32 drflac__decode_samples_with_residual__unencoded(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 count, drflac_uint8 unencodedBitsPerSample, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pSamplesOut) +{ + drflac_uint32 i; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(unencodedBitsPerSample <= 31); /* <-- unencodedBitsPerSample is a 5 bit number, so cannot exceed 31. */ + DRFLAC_ASSERT(pSamplesOut != NULL); + + for (i = 0; i < count; ++i) { + if (unencodedBitsPerSample > 0) { + if (!drflac__read_int32(bs, unencodedBitsPerSample, pSamplesOut + i)) { + return DRFLAC_FALSE; + } + } else { + pSamplesOut[i] = 0; + } + + if (drflac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { + pSamplesOut[i] += drflac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + } else { + pSamplesOut[i] += drflac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + } + } + + return DRFLAC_TRUE; +} + + +/* +Reads and decodes the residual for the sub-frame the decoder is currently sitting on. This function should be called +when the decoder is sitting at the very start of the RESIDUAL block. The first residuals will be ignored. The + and parameters are used to determine how many residual values need to be decoded. +*/ +static drflac_bool32 drflac__decode_samples_with_residual(drflac_bs* bs, drflac_uint32 bitsPerSample, drflac_uint32 blockSize, drflac_uint32 lpcOrder, drflac_int32 lpcShift, drflac_uint32 lpcPrecision, const drflac_int32* coefficients, drflac_int32* pDecodedSamples) +{ + drflac_uint8 residualMethod; + drflac_uint8 partitionOrder; + drflac_uint32 samplesInPartition; + drflac_uint32 partitionsRemaining; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(blockSize != 0); + DRFLAC_ASSERT(pDecodedSamples != NULL); /* <-- Should we allow NULL, in which case we just seek past the residual rather than do a full decode? */ + + if (!drflac__read_uint8(bs, 2, &residualMethod)) { + return DRFLAC_FALSE; + } + + if (residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { + return DRFLAC_FALSE; /* Unknown or unsupported residual coding method. */ + } + + /* Ignore the first values. */ + pDecodedSamples += lpcOrder; + + if (!drflac__read_uint8(bs, 4, &partitionOrder)) { + return DRFLAC_FALSE; + } + + /* + From the FLAC spec: + The Rice partition order in a Rice-coded residual section must be less than or equal to 8. + */ + if (partitionOrder > 8) { + return DRFLAC_FALSE; + } + + /* Validation check. */ + if ((blockSize / (1 << partitionOrder)) < lpcOrder) { + return DRFLAC_FALSE; + } + + samplesInPartition = (blockSize / (1 << partitionOrder)) - lpcOrder; + partitionsRemaining = (1 << partitionOrder); + for (;;) { + drflac_uint8 riceParam = 0; + if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) { + if (!drflac__read_uint8(bs, 4, &riceParam)) { + return DRFLAC_FALSE; + } + if (riceParam == 15) { + riceParam = 0xFF; + } + } else if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { + if (!drflac__read_uint8(bs, 5, &riceParam)) { + return DRFLAC_FALSE; + } + if (riceParam == 31) { + riceParam = 0xFF; + } + } + + if (riceParam != 0xFF) { + if (!drflac__decode_samples_with_residual__rice(bs, bitsPerSample, samplesInPartition, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { + return DRFLAC_FALSE; + } + } else { + drflac_uint8 unencodedBitsPerSample = 0; + if (!drflac__read_uint8(bs, 5, &unencodedBitsPerSample)) { + return DRFLAC_FALSE; + } + + if (!drflac__decode_samples_with_residual__unencoded(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { + return DRFLAC_FALSE; + } + } + + pDecodedSamples += samplesInPartition; + + if (partitionsRemaining == 1) { + break; + } + + partitionsRemaining -= 1; + + if (partitionOrder != 0) { + samplesInPartition = blockSize / (1 << partitionOrder); + } + } + + return DRFLAC_TRUE; +} + +/* +Reads and seeks past the residual for the sub-frame the decoder is currently sitting on. This function should be called +when the decoder is sitting at the very start of the RESIDUAL block. The first residuals will be set to 0. The + and parameters are used to determine how many residual values need to be decoded. +*/ +static drflac_bool32 drflac__read_and_seek_residual(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 order) +{ + drflac_uint8 residualMethod; + drflac_uint8 partitionOrder; + drflac_uint32 samplesInPartition; + drflac_uint32 partitionsRemaining; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(blockSize != 0); + + if (!drflac__read_uint8(bs, 2, &residualMethod)) { + return DRFLAC_FALSE; + } + + if (residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { + return DRFLAC_FALSE; /* Unknown or unsupported residual coding method. */ + } + + if (!drflac__read_uint8(bs, 4, &partitionOrder)) { + return DRFLAC_FALSE; + } + + /* + From the FLAC spec: + The Rice partition order in a Rice-coded residual section must be less than or equal to 8. + */ + if (partitionOrder > 8) { + return DRFLAC_FALSE; + } + + /* Validation check. */ + if ((blockSize / (1 << partitionOrder)) <= order) { + return DRFLAC_FALSE; + } + + samplesInPartition = (blockSize / (1 << partitionOrder)) - order; + partitionsRemaining = (1 << partitionOrder); + for (;;) + { + drflac_uint8 riceParam = 0; + if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) { + if (!drflac__read_uint8(bs, 4, &riceParam)) { + return DRFLAC_FALSE; + } + if (riceParam == 15) { + riceParam = 0xFF; + } + } else if (residualMethod == DRFLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { + if (!drflac__read_uint8(bs, 5, &riceParam)) { + return DRFLAC_FALSE; + } + if (riceParam == 31) { + riceParam = 0xFF; + } + } + + if (riceParam != 0xFF) { + if (!drflac__read_and_seek_residual__rice(bs, samplesInPartition, riceParam)) { + return DRFLAC_FALSE; + } + } else { + drflac_uint8 unencodedBitsPerSample = 0; + if (!drflac__read_uint8(bs, 5, &unencodedBitsPerSample)) { + return DRFLAC_FALSE; + } + + if (!drflac__seek_bits(bs, unencodedBitsPerSample * samplesInPartition)) { + return DRFLAC_FALSE; + } + } + + + if (partitionsRemaining == 1) { + break; + } + + partitionsRemaining -= 1; + samplesInPartition = blockSize / (1 << partitionOrder); + } + + return DRFLAC_TRUE; +} + + +static drflac_bool32 drflac__decode_samples__constant(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 subframeBitsPerSample, drflac_int32* pDecodedSamples) +{ + drflac_uint32 i; + + /* Only a single sample needs to be decoded here. */ + drflac_int32 sample; + if (!drflac__read_int32(bs, subframeBitsPerSample, &sample)) { + return DRFLAC_FALSE; + } + + /* + We don't really need to expand this, but it does simplify the process of reading samples. If this becomes a performance issue (unlikely) + we'll want to look at a more efficient way. + */ + for (i = 0; i < blockSize; ++i) { + pDecodedSamples[i] = sample; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples__verbatim(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 subframeBitsPerSample, drflac_int32* pDecodedSamples) +{ + drflac_uint32 i; + + for (i = 0; i < blockSize; ++i) { + drflac_int32 sample; + if (!drflac__read_int32(bs, subframeBitsPerSample, &sample)) { + return DRFLAC_FALSE; + } + + pDecodedSamples[i] = sample; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples__fixed(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 subframeBitsPerSample, drflac_uint8 lpcOrder, drflac_int32* pDecodedSamples) +{ + drflac_uint32 i; + + static drflac_int32 lpcCoefficientsTable[5][4] = { + {0, 0, 0, 0}, + {1, 0, 0, 0}, + {2, -1, 0, 0}, + {3, -3, 1, 0}, + {4, -6, 4, -1} + }; + + /* Warm up samples and coefficients. */ + for (i = 0; i < lpcOrder; ++i) { + drflac_int32 sample; + if (!drflac__read_int32(bs, subframeBitsPerSample, &sample)) { + return DRFLAC_FALSE; + } + + pDecodedSamples[i] = sample; + } + + if (!drflac__decode_samples_with_residual(bs, subframeBitsPerSample, blockSize, lpcOrder, 0, 4, lpcCoefficientsTable[lpcOrder], pDecodedSamples)) { + return DRFLAC_FALSE; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_samples__lpc(drflac_bs* bs, drflac_uint32 blockSize, drflac_uint32 bitsPerSample, drflac_uint8 lpcOrder, drflac_int32* pDecodedSamples) +{ + drflac_uint8 i; + drflac_uint8 lpcPrecision; + drflac_int8 lpcShift; + drflac_int32 coefficients[32]; + + /* Warm up samples. */ + for (i = 0; i < lpcOrder; ++i) { + drflac_int32 sample; + if (!drflac__read_int32(bs, bitsPerSample, &sample)) { + return DRFLAC_FALSE; + } + + pDecodedSamples[i] = sample; + } + + if (!drflac__read_uint8(bs, 4, &lpcPrecision)) { + return DRFLAC_FALSE; + } + if (lpcPrecision == 15) { + return DRFLAC_FALSE; /* Invalid. */ + } + lpcPrecision += 1; + + if (!drflac__read_int8(bs, 5, &lpcShift)) { + return DRFLAC_FALSE; + } + + /* + From the FLAC specification: + + Quantized linear predictor coefficient shift needed in bits (NOTE: this number is signed two's-complement) + + Emphasis on the "signed two's-complement". In practice there does not seem to be any encoders nor decoders supporting negative shifts. For now dr_flac is + not going to support negative shifts as I don't have any reference files. However, when a reference file comes through I will consider adding support. + */ + if (lpcShift < 0) { + return DRFLAC_FALSE; + } + + DRFLAC_ZERO_MEMORY(coefficients, sizeof(coefficients)); + for (i = 0; i < lpcOrder; ++i) { + if (!drflac__read_int32(bs, lpcPrecision, coefficients + i)) { + return DRFLAC_FALSE; + } + } + + if (!drflac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { + return DRFLAC_FALSE; + } + + return DRFLAC_TRUE; +} + + +static drflac_bool32 drflac__read_next_flac_frame_header(drflac_bs* bs, drflac_uint8 streaminfoBitsPerSample, drflac_frame_header* header) +{ + const drflac_uint32 sampleRateTable[12] = {0, 88200, 176400, 192000, 8000, 16000, 22050, 24000, 32000, 44100, 48000, 96000}; + const drflac_uint8 bitsPerSampleTable[8] = {0, 8, 12, (drflac_uint8)-1, 16, 20, 24, (drflac_uint8)-1}; /* -1 = reserved. */ + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(header != NULL); + + /* Keep looping until we find a valid sync code. */ + for (;;) { + drflac_uint8 crc8 = 0xCE; /* 0xCE = drflac_crc8(0, 0x3FFE, 14); */ + drflac_uint8 reserved = 0; + drflac_uint8 blockingStrategy = 0; + drflac_uint8 blockSize = 0; + drflac_uint8 sampleRate = 0; + drflac_uint8 channelAssignment = 0; + drflac_uint8 bitsPerSample = 0; + drflac_bool32 isVariableBlockSize; + + if (!drflac__find_and_seek_to_next_sync_code(bs)) { + return DRFLAC_FALSE; + } + + if (!drflac__read_uint8(bs, 1, &reserved)) { + return DRFLAC_FALSE; + } + if (reserved == 1) { + continue; + } + crc8 = drflac_crc8(crc8, reserved, 1); + + if (!drflac__read_uint8(bs, 1, &blockingStrategy)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, blockingStrategy, 1); + + if (!drflac__read_uint8(bs, 4, &blockSize)) { + return DRFLAC_FALSE; + } + if (blockSize == 0) { + continue; + } + crc8 = drflac_crc8(crc8, blockSize, 4); + + if (!drflac__read_uint8(bs, 4, &sampleRate)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, sampleRate, 4); + + if (!drflac__read_uint8(bs, 4, &channelAssignment)) { + return DRFLAC_FALSE; + } + if (channelAssignment > 10) { + continue; + } + crc8 = drflac_crc8(crc8, channelAssignment, 4); + + if (!drflac__read_uint8(bs, 3, &bitsPerSample)) { + return DRFLAC_FALSE; + } + if (bitsPerSample == 3 || bitsPerSample == 7) { + continue; + } + crc8 = drflac_crc8(crc8, bitsPerSample, 3); + + + if (!drflac__read_uint8(bs, 1, &reserved)) { + return DRFLAC_FALSE; + } + if (reserved == 1) { + continue; + } + crc8 = drflac_crc8(crc8, reserved, 1); + + + isVariableBlockSize = blockingStrategy == 1; + if (isVariableBlockSize) { + drflac_uint64 pcmFrameNumber; + drflac_result result = drflac__read_utf8_coded_number(bs, &pcmFrameNumber, &crc8); + if (result != DRFLAC_SUCCESS) { + if (result == DRFLAC_AT_END) { + return DRFLAC_FALSE; + } else { + continue; + } + } + header->flacFrameNumber = 0; + header->pcmFrameNumber = pcmFrameNumber; + } else { + drflac_uint64 flacFrameNumber = 0; + drflac_result result = drflac__read_utf8_coded_number(bs, &flacFrameNumber, &crc8); + if (result != DRFLAC_SUCCESS) { + if (result == DRFLAC_AT_END) { + return DRFLAC_FALSE; + } else { + continue; + } + } + header->flacFrameNumber = (drflac_uint32)flacFrameNumber; /* <-- Safe cast. */ + header->pcmFrameNumber = 0; + } + + + DRFLAC_ASSERT(blockSize > 0); + if (blockSize == 1) { + header->blockSizeInPCMFrames = 192; + } else if (blockSize <= 5) { + DRFLAC_ASSERT(blockSize >= 2); + header->blockSizeInPCMFrames = 576 * (1 << (blockSize - 2)); + } else if (blockSize == 6) { + if (!drflac__read_uint16(bs, 8, &header->blockSizeInPCMFrames)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, header->blockSizeInPCMFrames, 8); + header->blockSizeInPCMFrames += 1; + } else if (blockSize == 7) { + if (!drflac__read_uint16(bs, 16, &header->blockSizeInPCMFrames)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, header->blockSizeInPCMFrames, 16); + if (header->blockSizeInPCMFrames == 0xFFFF) { + return DRFLAC_FALSE; /* Frame is too big. This is the size of the frame minus 1. The STREAMINFO block defines the max block size which is 16-bits. Adding one will make it 17 bits and therefore too big. */ + } + header->blockSizeInPCMFrames += 1; + } else { + DRFLAC_ASSERT(blockSize >= 8); + header->blockSizeInPCMFrames = 256 * (1 << (blockSize - 8)); + } + + + if (sampleRate <= 11) { + header->sampleRate = sampleRateTable[sampleRate]; + } else if (sampleRate == 12) { + if (!drflac__read_uint32(bs, 8, &header->sampleRate)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, header->sampleRate, 8); + header->sampleRate *= 1000; + } else if (sampleRate == 13) { + if (!drflac__read_uint32(bs, 16, &header->sampleRate)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, header->sampleRate, 16); + } else if (sampleRate == 14) { + if (!drflac__read_uint32(bs, 16, &header->sampleRate)) { + return DRFLAC_FALSE; + } + crc8 = drflac_crc8(crc8, header->sampleRate, 16); + header->sampleRate *= 10; + } else { + continue; /* Invalid. Assume an invalid block. */ + } + + + header->channelAssignment = channelAssignment; + + header->bitsPerSample = bitsPerSampleTable[bitsPerSample]; + if (header->bitsPerSample == 0) { + header->bitsPerSample = streaminfoBitsPerSample; + } + + if (header->bitsPerSample != streaminfoBitsPerSample) { + /* If this subframe has a different bitsPerSample then streaminfo or the first frame, reject it */ + return DRFLAC_FALSE; + } + + if (!drflac__read_uint8(bs, 8, &header->crc8)) { + return DRFLAC_FALSE; + } + +#ifndef DR_FLAC_NO_CRC + if (header->crc8 != crc8) { + continue; /* CRC mismatch. Loop back to the top and find the next sync code. */ + } +#endif + return DRFLAC_TRUE; + } +} + +static drflac_bool32 drflac__read_subframe_header(drflac_bs* bs, drflac_subframe* pSubframe) +{ + drflac_uint8 header; + int type; + + if (!drflac__read_uint8(bs, 8, &header)) { + return DRFLAC_FALSE; + } + + /* First bit should always be 0. */ + if ((header & 0x80) != 0) { + return DRFLAC_FALSE; + } + + type = (header & 0x7E) >> 1; + if (type == 0) { + pSubframe->subframeType = DRFLAC_SUBFRAME_CONSTANT; + } else if (type == 1) { + pSubframe->subframeType = DRFLAC_SUBFRAME_VERBATIM; + } else { + if ((type & 0x20) != 0) { + pSubframe->subframeType = DRFLAC_SUBFRAME_LPC; + pSubframe->lpcOrder = (drflac_uint8)(type & 0x1F) + 1; + } else if ((type & 0x08) != 0) { + pSubframe->subframeType = DRFLAC_SUBFRAME_FIXED; + pSubframe->lpcOrder = (drflac_uint8)(type & 0x07); + if (pSubframe->lpcOrder > 4) { + pSubframe->subframeType = DRFLAC_SUBFRAME_RESERVED; + pSubframe->lpcOrder = 0; + } + } else { + pSubframe->subframeType = DRFLAC_SUBFRAME_RESERVED; + } + } + + if (pSubframe->subframeType == DRFLAC_SUBFRAME_RESERVED) { + return DRFLAC_FALSE; + } + + /* Wasted bits per sample. */ + pSubframe->wastedBitsPerSample = 0; + if ((header & 0x01) == 1) { + unsigned int wastedBitsPerSample; + if (!drflac__seek_past_next_set_bit(bs, &wastedBitsPerSample)) { + return DRFLAC_FALSE; + } + pSubframe->wastedBitsPerSample = (drflac_uint8)wastedBitsPerSample + 1; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_subframe(drflac_bs* bs, drflac_frame* frame, int subframeIndex, drflac_int32* pDecodedSamplesOut) +{ + drflac_subframe* pSubframe; + drflac_uint32 subframeBitsPerSample; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(frame != NULL); + + pSubframe = frame->subframes + subframeIndex; + if (!drflac__read_subframe_header(bs, pSubframe)) { + return DRFLAC_FALSE; + } + + /* Side channels require an extra bit per sample. Took a while to figure that one out... */ + subframeBitsPerSample = frame->header.bitsPerSample; + if ((frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) { + subframeBitsPerSample += 1; + } else if (frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) { + subframeBitsPerSample += 1; + } + + if (subframeBitsPerSample > 32) { + /* libFLAC and ffmpeg reject 33-bit subframes as well */ + return DRFLAC_FALSE; + } + + /* Need to handle wasted bits per sample. */ + if (pSubframe->wastedBitsPerSample >= subframeBitsPerSample) { + return DRFLAC_FALSE; + } + subframeBitsPerSample -= pSubframe->wastedBitsPerSample; + + pSubframe->pSamplesS32 = pDecodedSamplesOut; + + switch (pSubframe->subframeType) + { + case DRFLAC_SUBFRAME_CONSTANT: + { + drflac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + } break; + + case DRFLAC_SUBFRAME_VERBATIM: + { + drflac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + } break; + + case DRFLAC_SUBFRAME_FIXED: + { + drflac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + } break; + + case DRFLAC_SUBFRAME_LPC: + { + drflac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + } break; + + default: return DRFLAC_FALSE; + } + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__seek_subframe(drflac_bs* bs, drflac_frame* frame, int subframeIndex) +{ + drflac_subframe* pSubframe; + drflac_uint32 subframeBitsPerSample; + + DRFLAC_ASSERT(bs != NULL); + DRFLAC_ASSERT(frame != NULL); + + pSubframe = frame->subframes + subframeIndex; + if (!drflac__read_subframe_header(bs, pSubframe)) { + return DRFLAC_FALSE; + } + + /* Side channels require an extra bit per sample. Took a while to figure that one out... */ + subframeBitsPerSample = frame->header.bitsPerSample; + if ((frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) { + subframeBitsPerSample += 1; + } else if (frame->header.channelAssignment == DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) { + subframeBitsPerSample += 1; + } + + /* Need to handle wasted bits per sample. */ + if (pSubframe->wastedBitsPerSample >= subframeBitsPerSample) { + return DRFLAC_FALSE; + } + subframeBitsPerSample -= pSubframe->wastedBitsPerSample; + + pSubframe->pSamplesS32 = NULL; + + switch (pSubframe->subframeType) + { + case DRFLAC_SUBFRAME_CONSTANT: + { + if (!drflac__seek_bits(bs, subframeBitsPerSample)) { + return DRFLAC_FALSE; + } + } break; + + case DRFLAC_SUBFRAME_VERBATIM: + { + unsigned int bitsToSeek = frame->header.blockSizeInPCMFrames * subframeBitsPerSample; + if (!drflac__seek_bits(bs, bitsToSeek)) { + return DRFLAC_FALSE; + } + } break; + + case DRFLAC_SUBFRAME_FIXED: + { + unsigned int bitsToSeek = pSubframe->lpcOrder * subframeBitsPerSample; + if (!drflac__seek_bits(bs, bitsToSeek)) { + return DRFLAC_FALSE; + } + + if (!drflac__read_and_seek_residual(bs, frame->header.blockSizeInPCMFrames, pSubframe->lpcOrder)) { + return DRFLAC_FALSE; + } + } break; + + case DRFLAC_SUBFRAME_LPC: + { + drflac_uint8 lpcPrecision; + + unsigned int bitsToSeek = pSubframe->lpcOrder * subframeBitsPerSample; + if (!drflac__seek_bits(bs, bitsToSeek)) { + return DRFLAC_FALSE; + } + + if (!drflac__read_uint8(bs, 4, &lpcPrecision)) { + return DRFLAC_FALSE; + } + if (lpcPrecision == 15) { + return DRFLAC_FALSE; /* Invalid. */ + } + lpcPrecision += 1; + + + bitsToSeek = (pSubframe->lpcOrder * lpcPrecision) + 5; /* +5 for shift. */ + if (!drflac__seek_bits(bs, bitsToSeek)) { + return DRFLAC_FALSE; + } + + if (!drflac__read_and_seek_residual(bs, frame->header.blockSizeInPCMFrames, pSubframe->lpcOrder)) { + return DRFLAC_FALSE; + } + } break; + + default: return DRFLAC_FALSE; + } + + return DRFLAC_TRUE; +} + + +static DRFLAC_INLINE drflac_uint8 drflac__get_channel_count_from_channel_assignment(drflac_int8 channelAssignment) +{ + drflac_uint8 lookup[] = {1, 2, 3, 4, 5, 6, 7, 8, 2, 2, 2}; + + DRFLAC_ASSERT(channelAssignment <= 10); + return lookup[channelAssignment]; +} + +static drflac_result drflac__decode_flac_frame(drflac* pFlac) +{ + int channelCount; + int i; + drflac_uint8 paddingSizeInBits; + drflac_uint16 desiredCRC16; +#ifndef DR_FLAC_NO_CRC + drflac_uint16 actualCRC16; +#endif + + /* This function should be called while the stream is sitting on the first byte after the frame header. */ + DRFLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); + + /* The frame block size must never be larger than the maximum block size defined by the FLAC stream. */ + if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->maxBlockSizeInPCMFrames) { + return DRFLAC_ERROR; + } + + /* The number of channels in the frame must match the channel count from the STREAMINFO block. */ + channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + if (channelCount != (int)pFlac->channels) { + return DRFLAC_ERROR; + } + + for (i = 0; i < channelCount; ++i) { + if (!drflac__decode_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i, pFlac->pDecodedSamples + (pFlac->currentFLACFrame.header.blockSizeInPCMFrames * i))) { + return DRFLAC_ERROR; + } + } + + paddingSizeInBits = (drflac_uint8)(DRFLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7); + if (paddingSizeInBits > 0) { + drflac_uint8 padding = 0; + if (!drflac__read_uint8(&pFlac->bs, paddingSizeInBits, &padding)) { + return DRFLAC_AT_END; + } + } + +#ifndef DR_FLAC_NO_CRC + actualCRC16 = drflac__flush_crc16(&pFlac->bs); +#endif + if (!drflac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) { + return DRFLAC_AT_END; + } + +#ifndef DR_FLAC_NO_CRC + if (actualCRC16 != desiredCRC16) { + return DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ + } +#endif + + pFlac->currentFLACFrame.pcmFramesRemaining = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; + + return DRFLAC_SUCCESS; +} + +static drflac_result drflac__seek_flac_frame(drflac* pFlac) +{ + int channelCount; + int i; + drflac_uint16 desiredCRC16; +#ifndef DR_FLAC_NO_CRC + drflac_uint16 actualCRC16; +#endif + + channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + for (i = 0; i < channelCount; ++i) { + if (!drflac__seek_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i)) { + return DRFLAC_ERROR; + } + } + + /* Padding. */ + if (!drflac__seek_bits(&pFlac->bs, DRFLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7)) { + return DRFLAC_ERROR; + } + + /* CRC. */ +#ifndef DR_FLAC_NO_CRC + actualCRC16 = drflac__flush_crc16(&pFlac->bs); +#endif + if (!drflac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) { + return DRFLAC_AT_END; + } + +#ifndef DR_FLAC_NO_CRC + if (actualCRC16 != desiredCRC16) { + return DRFLAC_CRC_MISMATCH; /* CRC mismatch. */ + } +#endif + + return DRFLAC_SUCCESS; +} + +static drflac_bool32 drflac__read_and_decode_next_flac_frame(drflac* pFlac) +{ + DRFLAC_ASSERT(pFlac != NULL); + + for (;;) { + drflac_result result; + + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + + result = drflac__decode_flac_frame(pFlac); + if (result != DRFLAC_SUCCESS) { + if (result == DRFLAC_CRC_MISMATCH) { + continue; /* CRC mismatch. Skip to the next frame. */ + } else { + return DRFLAC_FALSE; + } + } + + return DRFLAC_TRUE; + } +} + +static void drflac__get_pcm_frame_range_of_current_flac_frame(drflac* pFlac, drflac_uint64* pFirstPCMFrame, drflac_uint64* pLastPCMFrame) +{ + drflac_uint64 firstPCMFrame; + drflac_uint64 lastPCMFrame; + + DRFLAC_ASSERT(pFlac != NULL); + + firstPCMFrame = pFlac->currentFLACFrame.header.pcmFrameNumber; + if (firstPCMFrame == 0) { + firstPCMFrame = ((drflac_uint64)pFlac->currentFLACFrame.header.flacFrameNumber) * pFlac->maxBlockSizeInPCMFrames; + } + + lastPCMFrame = firstPCMFrame + pFlac->currentFLACFrame.header.blockSizeInPCMFrames; + if (lastPCMFrame > 0) { + lastPCMFrame -= 1; /* Needs to be zero based. */ + } + + if (pFirstPCMFrame) { + *pFirstPCMFrame = firstPCMFrame; + } + if (pLastPCMFrame) { + *pLastPCMFrame = lastPCMFrame; + } +} + +static drflac_bool32 drflac__seek_to_first_frame(drflac* pFlac) +{ + drflac_bool32 result; + + DRFLAC_ASSERT(pFlac != NULL); + + result = drflac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes); + + DRFLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame)); + pFlac->currentPCMFrame = 0; + + return result; +} + +static DRFLAC_INLINE drflac_result drflac__seek_to_next_flac_frame(drflac* pFlac) +{ + /* This function should only ever be called while the decoder is sitting on the first byte past the FRAME_HEADER section. */ + DRFLAC_ASSERT(pFlac != NULL); + return drflac__seek_flac_frame(pFlac); +} + + +static drflac_uint64 drflac__seek_forward_by_pcm_frames(drflac* pFlac, drflac_uint64 pcmFramesToSeek) +{ + drflac_uint64 pcmFramesRead = 0; + while (pcmFramesToSeek > 0) { + if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_and_decode_next_flac_frame(pFlac)) { + break; /* Couldn't read the next frame, so just break from the loop and return. */ + } + } else { + if (pFlac->currentFLACFrame.pcmFramesRemaining > pcmFramesToSeek) { + pcmFramesRead += pcmFramesToSeek; + pFlac->currentFLACFrame.pcmFramesRemaining -= (drflac_uint32)pcmFramesToSeek; /* <-- Safe cast. Will always be < currentFrame.pcmFramesRemaining < 65536. */ + pcmFramesToSeek = 0; + } else { + pcmFramesRead += pFlac->currentFLACFrame.pcmFramesRemaining; + pcmFramesToSeek -= pFlac->currentFLACFrame.pcmFramesRemaining; + pFlac->currentFLACFrame.pcmFramesRemaining = 0; + } + } + } + + pFlac->currentPCMFrame += pcmFramesRead; + return pcmFramesRead; +} + + +static drflac_bool32 drflac__seek_to_pcm_frame__brute_force(drflac* pFlac, drflac_uint64 pcmFrameIndex) +{ + drflac_bool32 isMidFrame = DRFLAC_FALSE; + drflac_uint64 runningPCMFrameCount; + + DRFLAC_ASSERT(pFlac != NULL); + + /* If we are seeking forward we start from the current position. Otherwise we need to start all the way from the start of the file. */ + if (pcmFrameIndex >= pFlac->currentPCMFrame) { + /* Seeking forward. Need to seek from the current position. */ + runningPCMFrameCount = pFlac->currentPCMFrame; + + /* The frame header for the first frame may not yet have been read. We need to do that if necessary. */ + if (pFlac->currentPCMFrame == 0 && pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } else { + isMidFrame = DRFLAC_TRUE; + } + } else { + /* Seeking backwards. Need to seek from the start of the file. */ + runningPCMFrameCount = 0; + + /* Move back to the start. */ + if (!drflac__seek_to_first_frame(pFlac)) { + return DRFLAC_FALSE; + } + + /* Decode the first frame in preparation for sample-exact seeking below. */ + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } + + /* + We need to as quickly as possible find the frame that contains the target sample. To do this, we iterate over each frame and inspect its + header. If based on the header we can determine that the frame contains the sample, we do a full decode of that frame. + */ + for (;;) { + drflac_uint64 pcmFrameCountInThisFLACFrame; + drflac_uint64 firstPCMFrameInFLACFrame = 0; + drflac_uint64 lastPCMFrameInFLACFrame = 0; + + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame); + + pcmFrameCountInThisFLACFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1; + if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFLACFrame)) { + /* + The sample should be in this frame. We need to fully decode it, however if it's an invalid frame (a CRC mismatch), we need to pretend + it never existed and keep iterating. + */ + drflac_uint64 pcmFramesToDecode = pcmFrameIndex - runningPCMFrameCount; + + if (!isMidFrame) { + drflac_result result = drflac__decode_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode; /* <-- If this fails, something bad has happened (it should never fail). */ + } else { + if (result == DRFLAC_CRC_MISMATCH) { + goto next_iteration; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } else { + /* We started seeking mid-frame which means we need to skip the frame decoding part. */ + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode; + } + } else { + /* + It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this + frame never existed and leave the running sample count untouched. + */ + if (!isMidFrame) { + drflac_result result = drflac__seek_to_next_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + runningPCMFrameCount += pcmFrameCountInThisFLACFrame; + } else { + if (result == DRFLAC_CRC_MISMATCH) { + goto next_iteration; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } else { + /* + We started seeking mid-frame which means we need to seek by reading to the end of the frame instead of with + drflac__seek_to_next_flac_frame() which only works if the decoder is sitting on the byte just after the frame header. + */ + runningPCMFrameCount += pFlac->currentFLACFrame.pcmFramesRemaining; + pFlac->currentFLACFrame.pcmFramesRemaining = 0; + isMidFrame = DRFLAC_FALSE; + } + + /* If we are seeking to the end of the file and we've just hit it, we're done. */ + if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) { + return DRFLAC_TRUE; + } + } + + next_iteration: + /* Grab the next frame in preparation for the next iteration. */ + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } +} + + +#if !defined(DR_FLAC_NO_CRC) +/* +We use an average compression ratio to determine our approximate start location. FLAC files are generally about 50%-70% the size of their +uncompressed counterparts so we'll use this as a basis. I'm going to split the middle and use a factor of 0.6 to determine the starting +location. +*/ +#define DRFLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO 0.6f + +static drflac_bool32 drflac__seek_to_approximate_flac_frame_to_byte(drflac* pFlac, drflac_uint64 targetByte, drflac_uint64 rangeLo, drflac_uint64 rangeHi, drflac_uint64* pLastSuccessfulSeekOffset) +{ + DRFLAC_ASSERT(pFlac != NULL); + DRFLAC_ASSERT(pLastSuccessfulSeekOffset != NULL); + DRFLAC_ASSERT(targetByte >= rangeLo); + DRFLAC_ASSERT(targetByte <= rangeHi); + + *pLastSuccessfulSeekOffset = pFlac->firstFLACFramePosInBytes; + + for (;;) { + /* After rangeLo == rangeHi == targetByte fails, we need to break out. */ + drflac_uint64 lastTargetByte = targetByte; + + /* When seeking to a byte, failure probably means we've attempted to seek beyond the end of the stream. To counter this we just halve it each attempt. */ + if (!drflac__seek_to_byte(&pFlac->bs, targetByte)) { + /* If we couldn't even seek to the first byte in the stream we have a problem. Just abandon the whole thing. */ + if (targetByte == 0) { + drflac__seek_to_first_frame(pFlac); /* Try to recover. */ + return DRFLAC_FALSE; + } + + /* Halve the byte location and continue. */ + targetByte = rangeLo + ((rangeHi - rangeLo)/2); + rangeHi = targetByte; + } else { + /* Getting here should mean that we have seeked to an appropriate byte. */ + + /* Clear the details of the FLAC frame so we don't misreport data. */ + DRFLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame)); + + /* + Now seek to the next FLAC frame. We need to decode the entire frame (not just the header) because it's possible for the header to incorrectly pass the + CRC check and return bad data. We need to decode the entire frame to be more certain. Although this seems unlikely, this has happened to me in testing + so it needs to stay this way for now. + */ +#if 1 + if (!drflac__read_and_decode_next_flac_frame(pFlac)) { + /* Halve the byte location and continue. */ + targetByte = rangeLo + ((rangeHi - rangeLo)/2); + rangeHi = targetByte; + } else { + break; + } +#else + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + /* Halve the byte location and continue. */ + targetByte = rangeLo + ((rangeHi - rangeLo)/2); + rangeHi = targetByte; + } else { + break; + } +#endif + } + + /* We already tried this byte and there are no more to try, break out. */ + if(targetByte == lastTargetByte) { + return DRFLAC_FALSE; + } + } + + /* The current PCM frame needs to be updated based on the frame we just seeked to. */ + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL); + + DRFLAC_ASSERT(targetByte <= rangeHi); + + *pLastSuccessfulSeekOffset = targetByte; + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__decode_flac_frame_and_seek_forward_by_pcm_frames(drflac* pFlac, drflac_uint64 offset) +{ + /* This section of code would be used if we were only decoding the FLAC frame header when calling drflac__seek_to_approximate_flac_frame_to_byte(). */ +#if 0 + if (drflac__decode_flac_frame(pFlac) != DRFLAC_SUCCESS) { + /* We failed to decode this frame which may be due to it being corrupt. We'll just use the next valid FLAC frame. */ + if (drflac__read_and_decode_next_flac_frame(pFlac) == DRFLAC_FALSE) { + return DRFLAC_FALSE; + } + } +#endif + + return drflac__seek_forward_by_pcm_frames(pFlac, offset) == offset; +} + + +static drflac_bool32 drflac__seek_to_pcm_frame__binary_search_internal(drflac* pFlac, drflac_uint64 pcmFrameIndex, drflac_uint64 byteRangeLo, drflac_uint64 byteRangeHi) +{ + /* This assumes pFlac->currentPCMFrame is sitting on byteRangeLo upon entry. */ + + drflac_uint64 targetByte; + drflac_uint64 pcmRangeLo = pFlac->totalPCMFrameCount; + drflac_uint64 pcmRangeHi = 0; + drflac_uint64 lastSuccessfulSeekOffset = (drflac_uint64)-1; + drflac_uint64 closestSeekOffsetBeforeTargetPCMFrame = byteRangeLo; + drflac_uint32 seekForwardThreshold = (pFlac->maxBlockSizeInPCMFrames != 0) ? pFlac->maxBlockSizeInPCMFrames*2 : 4096; + + targetByte = byteRangeLo + (drflac_uint64)(((drflac_int64)((pcmFrameIndex - pFlac->currentPCMFrame) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * DRFLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO); + if (targetByte > byteRangeHi) { + targetByte = byteRangeHi; + } + + for (;;) { + if (drflac__seek_to_approximate_flac_frame_to_byte(pFlac, targetByte, byteRangeLo, byteRangeHi, &lastSuccessfulSeekOffset)) { + /* We found a FLAC frame. We need to check if it contains the sample we're looking for. */ + drflac_uint64 newPCMRangeLo; + drflac_uint64 newPCMRangeHi; + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &newPCMRangeLo, &newPCMRangeHi); + + /* If we selected the same frame, it means we should be pretty close. Just decode the rest. */ + if (pcmRangeLo == newPCMRangeLo) { + if (!drflac__seek_to_approximate_flac_frame_to_byte(pFlac, closestSeekOffsetBeforeTargetPCMFrame, closestSeekOffsetBeforeTargetPCMFrame, byteRangeHi, &lastSuccessfulSeekOffset)) { + break; /* Failed to seek to closest frame. */ + } + + if (drflac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) { + return DRFLAC_TRUE; + } else { + break; /* Failed to seek forward. */ + } + } + + pcmRangeLo = newPCMRangeLo; + pcmRangeHi = newPCMRangeHi; + + if (pcmRangeLo <= pcmFrameIndex && pcmRangeHi >= pcmFrameIndex) { + /* The target PCM frame is in this FLAC frame. */ + if (drflac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame) ) { + return DRFLAC_TRUE; + } else { + break; /* Failed to seek to FLAC frame. */ + } + } else { + const float approxCompressionRatio = (drflac_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((drflac_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0f); + + if (pcmRangeLo > pcmFrameIndex) { + /* We seeked too far forward. We need to move our target byte backward and try again. */ + byteRangeHi = lastSuccessfulSeekOffset; + if (byteRangeLo > byteRangeHi) { + byteRangeLo = byteRangeHi; + } + + targetByte = byteRangeLo + ((byteRangeHi - byteRangeLo) / 2); + if (targetByte < byteRangeLo) { + targetByte = byteRangeLo; + } + } else /*if (pcmRangeHi < pcmFrameIndex)*/ { + /* We didn't seek far enough. We need to move our target byte forward and try again. */ + + /* If we're close enough we can just seek forward. */ + if ((pcmFrameIndex - pcmRangeLo) < seekForwardThreshold) { + if (drflac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) { + return DRFLAC_TRUE; + } else { + break; /* Failed to seek to FLAC frame. */ + } + } else { + byteRangeLo = lastSuccessfulSeekOffset; + if (byteRangeHi < byteRangeLo) { + byteRangeHi = byteRangeLo; + } + + targetByte = lastSuccessfulSeekOffset + (drflac_uint64)(((drflac_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * approxCompressionRatio); + if (targetByte > byteRangeHi) { + targetByte = byteRangeHi; + } + + if (closestSeekOffsetBeforeTargetPCMFrame < lastSuccessfulSeekOffset) { + closestSeekOffsetBeforeTargetPCMFrame = lastSuccessfulSeekOffset; + } + } + } + } + } else { + /* Getting here is really bad. We just recover as best we can, but moving to the first frame in the stream, and then abort. */ + break; + } + } + + drflac__seek_to_first_frame(pFlac); /* <-- Try to recover. */ + return DRFLAC_FALSE; +} + +static drflac_bool32 drflac__seek_to_pcm_frame__binary_search(drflac* pFlac, drflac_uint64 pcmFrameIndex) +{ + drflac_uint64 byteRangeLo; + drflac_uint64 byteRangeHi; + drflac_uint32 seekForwardThreshold = (pFlac->maxBlockSizeInPCMFrames != 0) ? pFlac->maxBlockSizeInPCMFrames*2 : 4096; + + /* Our algorithm currently assumes the FLAC stream is currently sitting at the start. */ + if (drflac__seek_to_first_frame(pFlac) == DRFLAC_FALSE) { + return DRFLAC_FALSE; + } + + /* If we're close enough to the start, just move to the start and seek forward. */ + if (pcmFrameIndex < seekForwardThreshold) { + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFrameIndex) == pcmFrameIndex; + } + + /* + Our starting byte range is the byte position of the first FLAC frame and the approximate end of the file as if it were completely uncompressed. This ensures + the entire file is included, even though most of the time it'll exceed the end of the actual stream. This is OK as the frame searching logic will handle it. + */ + byteRangeLo = pFlac->firstFLACFramePosInBytes; + byteRangeHi = pFlac->firstFLACFramePosInBytes + (drflac_uint64)((drflac_int64)(pFlac->totalPCMFrameCount * pFlac->channels * pFlac->bitsPerSample)/8.0f); + + return drflac__seek_to_pcm_frame__binary_search_internal(pFlac, pcmFrameIndex, byteRangeLo, byteRangeHi); +} +#endif /* !DR_FLAC_NO_CRC */ + +static drflac_bool32 drflac__seek_to_pcm_frame__seek_table(drflac* pFlac, drflac_uint64 pcmFrameIndex) +{ + drflac_uint32 iClosestSeekpoint = 0; + drflac_bool32 isMidFrame = DRFLAC_FALSE; + drflac_uint64 runningPCMFrameCount; + drflac_uint32 iSeekpoint; + + + DRFLAC_ASSERT(pFlac != NULL); + + if (pFlac->pSeekpoints == NULL || pFlac->seekpointCount == 0) { + return DRFLAC_FALSE; + } + + /* Do not use the seektable if pcmFramIndex is not coverd by it. */ + if (pFlac->pSeekpoints[0].firstPCMFrame > pcmFrameIndex) { + return DRFLAC_FALSE; + } + + for (iSeekpoint = 0; iSeekpoint < pFlac->seekpointCount; ++iSeekpoint) { + if (pFlac->pSeekpoints[iSeekpoint].firstPCMFrame >= pcmFrameIndex) { + break; + } + + iClosestSeekpoint = iSeekpoint; + } + + /* There's been cases where the seek table contains only zeros. We need to do some basic validation on the closest seekpoint. */ + if (pFlac->pSeekpoints[iClosestSeekpoint].pcmFrameCount == 0 || pFlac->pSeekpoints[iClosestSeekpoint].pcmFrameCount > pFlac->maxBlockSizeInPCMFrames) { + return DRFLAC_FALSE; + } + if (pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame > pFlac->totalPCMFrameCount && pFlac->totalPCMFrameCount > 0) { + return DRFLAC_FALSE; + } + +#if !defined(DR_FLAC_NO_CRC) + /* At this point we should know the closest seek point. We can use a binary search for this. We need to know the total sample count for this. */ + if (pFlac->totalPCMFrameCount > 0) { + drflac_uint64 byteRangeLo; + drflac_uint64 byteRangeHi; + + byteRangeHi = pFlac->firstFLACFramePosInBytes + (drflac_uint64)((drflac_int64)(pFlac->totalPCMFrameCount * pFlac->channels * pFlac->bitsPerSample)/8.0f); + byteRangeLo = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset; + + /* + If our closest seek point is not the last one, we only need to search between it and the next one. The section below calculates an appropriate starting + value for byteRangeHi which will clamp it appropriately. + + Note that the next seekpoint must have an offset greater than the closest seekpoint because otherwise our binary search algorithm will break down. There + have been cases where a seektable consists of seek points where every byte offset is set to 0 which causes problems. If this happens we need to abort. + */ + if (iClosestSeekpoint < pFlac->seekpointCount-1) { + drflac_uint32 iNextSeekpoint = iClosestSeekpoint + 1; + + /* Basic validation on the seekpoints to ensure they're usable. */ + if (pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset >= pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset || pFlac->pSeekpoints[iNextSeekpoint].pcmFrameCount == 0) { + return DRFLAC_FALSE; /* The next seekpoint doesn't look right. The seek table cannot be trusted from here. Abort. */ + } + + if (pFlac->pSeekpoints[iNextSeekpoint].firstPCMFrame != (((drflac_uint64)0xFFFFFFFF << 32) | 0xFFFFFFFF)) { /* Make sure it's not a placeholder seekpoint. */ + byteRangeHi = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset - 1; /* byteRangeHi must be zero based. */ + } + } + + if (drflac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset)) { + if (drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL); + + if (drflac__seek_to_pcm_frame__binary_search_internal(pFlac, pcmFrameIndex, byteRangeLo, byteRangeHi)) { + return DRFLAC_TRUE; + } + } + } + } +#endif /* !DR_FLAC_NO_CRC */ + + /* Getting here means we need to use a slower algorithm because the binary search method failed or cannot be used. */ + + /* + If we are seeking forward and the closest seekpoint is _before_ the current sample, we just seek forward from where we are. Otherwise we start seeking + from the seekpoint's first sample. + */ + if (pcmFrameIndex >= pFlac->currentPCMFrame && pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame <= pFlac->currentPCMFrame) { + /* Optimized case. Just seek forward from where we are. */ + runningPCMFrameCount = pFlac->currentPCMFrame; + + /* The frame header for the first frame may not yet have been read. We need to do that if necessary. */ + if (pFlac->currentPCMFrame == 0 && pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } else { + isMidFrame = DRFLAC_TRUE; + } + } else { + /* Slower case. Seek to the start of the seekpoint and then seek forward from there. */ + runningPCMFrameCount = pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame; + + if (!drflac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset)) { + return DRFLAC_FALSE; + } + + /* Grab the frame the seekpoint is sitting on in preparation for the sample-exact seeking below. */ + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } + + for (;;) { + drflac_uint64 pcmFrameCountInThisFLACFrame; + drflac_uint64 firstPCMFrameInFLACFrame = 0; + drflac_uint64 lastPCMFrameInFLACFrame = 0; + + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame); + + pcmFrameCountInThisFLACFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1; + if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFLACFrame)) { + /* + The sample should be in this frame. We need to fully decode it, but if it's an invalid frame (a CRC mismatch) we need to pretend + it never existed and keep iterating. + */ + drflac_uint64 pcmFramesToDecode = pcmFrameIndex - runningPCMFrameCount; + + if (!isMidFrame) { + drflac_result result = drflac__decode_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode; /* <-- If this fails, something bad has happened (it should never fail). */ + } else { + if (result == DRFLAC_CRC_MISMATCH) { + goto next_iteration; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } else { + /* We started seeking mid-frame which means we need to skip the frame decoding part. */ + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode; + } + } else { + /* + It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this + frame never existed and leave the running sample count untouched. + */ + if (!isMidFrame) { + drflac_result result = drflac__seek_to_next_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + runningPCMFrameCount += pcmFrameCountInThisFLACFrame; + } else { + if (result == DRFLAC_CRC_MISMATCH) { + goto next_iteration; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } else { + /* + We started seeking mid-frame which means we need to seek by reading to the end of the frame instead of with + drflac__seek_to_next_flac_frame() which only works if the decoder is sitting on the byte just after the frame header. + */ + runningPCMFrameCount += pFlac->currentFLACFrame.pcmFramesRemaining; + pFlac->currentFLACFrame.pcmFramesRemaining = 0; + isMidFrame = DRFLAC_FALSE; + } + + /* If we are seeking to the end of the file and we've just hit it, we're done. */ + if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) { + return DRFLAC_TRUE; + } + } + + next_iteration: + /* Grab the next frame in preparation for the next iteration. */ + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + } +} + + +#ifndef DR_FLAC_NO_OGG +typedef struct +{ + drflac_uint8 capturePattern[4]; /* Should be "OggS" */ + drflac_uint8 structureVersion; /* Always 0. */ + drflac_uint8 headerType; + drflac_uint64 granulePosition; + drflac_uint32 serialNumber; + drflac_uint32 sequenceNumber; + drflac_uint32 checksum; + drflac_uint8 segmentCount; + drflac_uint8 segmentTable[255]; +} drflac_ogg_page_header; +#endif + +typedef struct +{ + drflac_read_proc onRead; + drflac_seek_proc onSeek; + drflac_meta_proc onMeta; + drflac_container container; + void* pUserData; + void* pUserDataMD; + drflac_uint32 sampleRate; + drflac_uint8 channels; + drflac_uint8 bitsPerSample; + drflac_uint64 totalPCMFrameCount; + drflac_uint16 maxBlockSizeInPCMFrames; + drflac_uint64 runningFilePos; + drflac_bool32 hasStreamInfoBlock; + drflac_bool32 hasMetadataBlocks; + drflac_bs bs; /* <-- A bit streamer is required for loading data during initialization. */ + drflac_frame_header firstFrameHeader; /* <-- The header of the first frame that was read during relaxed initalization. Only set if there is no STREAMINFO block. */ + +#ifndef DR_FLAC_NO_OGG + drflac_uint32 oggSerial; + drflac_uint64 oggFirstBytePos; + drflac_ogg_page_header oggBosHeader; +#endif +} drflac_init_info; + +static DRFLAC_INLINE void drflac__decode_block_header(drflac_uint32 blockHeader, drflac_uint8* isLastBlock, drflac_uint8* blockType, drflac_uint32* blockSize) +{ + blockHeader = drflac__be2host_32(blockHeader); + *isLastBlock = (drflac_uint8)((blockHeader & 0x80000000UL) >> 31); + *blockType = (drflac_uint8)((blockHeader & 0x7F000000UL) >> 24); + *blockSize = (blockHeader & 0x00FFFFFFUL); +} + +static DRFLAC_INLINE drflac_bool32 drflac__read_and_decode_block_header(drflac_read_proc onRead, void* pUserData, drflac_uint8* isLastBlock, drflac_uint8* blockType, drflac_uint32* blockSize) +{ + drflac_uint32 blockHeader; + + *blockSize = 0; + if (onRead(pUserData, &blockHeader, 4) != 4) { + return DRFLAC_FALSE; + } + + drflac__decode_block_header(blockHeader, isLastBlock, blockType, blockSize); + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__read_streaminfo(drflac_read_proc onRead, void* pUserData, drflac_streaminfo* pStreamInfo) +{ + drflac_uint32 blockSizes; + drflac_uint64 frameSizes = 0; + drflac_uint64 importantProps; + drflac_uint8 md5[16]; + + /* min/max block size. */ + if (onRead(pUserData, &blockSizes, 4) != 4) { + return DRFLAC_FALSE; + } + + /* min/max frame size. */ + if (onRead(pUserData, &frameSizes, 6) != 6) { + return DRFLAC_FALSE; + } + + /* Sample rate, channels, bits per sample and total sample count. */ + if (onRead(pUserData, &importantProps, 8) != 8) { + return DRFLAC_FALSE; + } + + /* MD5 */ + if (onRead(pUserData, md5, sizeof(md5)) != sizeof(md5)) { + return DRFLAC_FALSE; + } + + blockSizes = drflac__be2host_32(blockSizes); + frameSizes = drflac__be2host_64(frameSizes); + importantProps = drflac__be2host_64(importantProps); + + pStreamInfo->minBlockSizeInPCMFrames = (drflac_uint16)((blockSizes & 0xFFFF0000) >> 16); + pStreamInfo->maxBlockSizeInPCMFrames = (drflac_uint16) (blockSizes & 0x0000FFFF); + pStreamInfo->minFrameSizeInPCMFrames = (drflac_uint32)((frameSizes & (((drflac_uint64)0x00FFFFFF << 16) << 24)) >> 40); + pStreamInfo->maxFrameSizeInPCMFrames = (drflac_uint32)((frameSizes & (((drflac_uint64)0x00FFFFFF << 16) << 0)) >> 16); + pStreamInfo->sampleRate = (drflac_uint32)((importantProps & (((drflac_uint64)0x000FFFFF << 16) << 28)) >> 44); + pStreamInfo->channels = (drflac_uint8 )((importantProps & (((drflac_uint64)0x0000000E << 16) << 24)) >> 41) + 1; + pStreamInfo->bitsPerSample = (drflac_uint8 )((importantProps & (((drflac_uint64)0x0000001F << 16) << 20)) >> 36) + 1; + pStreamInfo->totalPCMFrameCount = ((importantProps & ((((drflac_uint64)0x0000000F << 16) << 16) | 0xFFFFFFFF))); + DRFLAC_COPY_MEMORY(pStreamInfo->md5, md5, sizeof(md5)); + + return DRFLAC_TRUE; +} + + +static void* drflac__malloc_default(size_t sz, void* pUserData) +{ + (void)pUserData; + return DRFLAC_MALLOC(sz); +} + +static void* drflac__realloc_default(void* p, size_t sz, void* pUserData) +{ + (void)pUserData; + return DRFLAC_REALLOC(p, sz); +} + +static void drflac__free_default(void* p, void* pUserData) +{ + (void)pUserData; + DRFLAC_FREE(p); +} + + +static void* drflac__malloc_from_callbacks(size_t sz, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + if (pAllocationCallbacks == NULL) { + return NULL; + } + + if (pAllocationCallbacks->onMalloc != NULL) { + return pAllocationCallbacks->onMalloc(sz, pAllocationCallbacks->pUserData); + } + + /* Try using realloc(). */ + if (pAllocationCallbacks->onRealloc != NULL) { + return pAllocationCallbacks->onRealloc(NULL, sz, pAllocationCallbacks->pUserData); + } + + return NULL; +} + +static void* drflac__realloc_from_callbacks(void* p, size_t szNew, size_t szOld, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + if (pAllocationCallbacks == NULL) { + return NULL; + } + + if (pAllocationCallbacks->onRealloc != NULL) { + return pAllocationCallbacks->onRealloc(p, szNew, pAllocationCallbacks->pUserData); + } + + /* Try emulating realloc() in terms of malloc()/free(). */ + if (pAllocationCallbacks->onMalloc != NULL && pAllocationCallbacks->onFree != NULL) { + void* p2; + + p2 = pAllocationCallbacks->onMalloc(szNew, pAllocationCallbacks->pUserData); + if (p2 == NULL) { + return NULL; + } + + if (p != NULL) { + DRFLAC_COPY_MEMORY(p2, p, szOld); + pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData); + } + + return p2; + } + + return NULL; +} + +static void drflac__free_from_callbacks(void* p, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + if (p == NULL || pAllocationCallbacks == NULL) { + return; + } + + if (pAllocationCallbacks->onFree != NULL) { + pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData); + } +} + + +static drflac_bool32 drflac__read_and_decode_metadata(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, void* pUserDataMD, drflac_uint64* pFirstFramePos, drflac_uint64* pSeektablePos, drflac_uint32* pSeekpointCount, drflac_allocation_callbacks* pAllocationCallbacks) +{ + /* + We want to keep track of the byte position in the stream of the seektable. At the time of calling this function we know that + we'll be sitting on byte 42. + */ + drflac_uint64 runningFilePos = 42; + drflac_uint64 seektablePos = 0; + drflac_uint32 seektableSize = 0; + + for (;;) { + drflac_metadata metadata; + drflac_uint8 isLastBlock = 0; + drflac_uint8 blockType = 0; + drflac_uint32 blockSize; + if (drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize) == DRFLAC_FALSE) { + return DRFLAC_FALSE; + } + runningFilePos += 4; + + metadata.type = blockType; + metadata.pRawData = NULL; + metadata.rawDataSize = 0; + + switch (blockType) + { + case DRFLAC_METADATA_BLOCK_TYPE_APPLICATION: + { + if (blockSize < 4) { + return DRFLAC_FALSE; + } + + if (onMeta) { + void* pRawData = drflac__malloc_from_callbacks(blockSize, pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + if (onRead(pUserData, pRawData, blockSize) != blockSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + metadata.data.application.id = drflac__be2host_32(*(drflac_uint32*)pRawData); + metadata.data.application.pData = (const void*)((drflac_uint8*)pRawData + sizeof(drflac_uint32)); + metadata.data.application.dataSize = blockSize - sizeof(drflac_uint32); + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_SEEKTABLE: + { + seektablePos = runningFilePos; + seektableSize = blockSize; + + if (onMeta) { + drflac_uint32 seekpointCount; + drflac_uint32 iSeekpoint; + void* pRawData; + + seekpointCount = blockSize/DRFLAC_SEEKPOINT_SIZE_IN_BYTES; + + pRawData = drflac__malloc_from_callbacks(seekpointCount * sizeof(drflac_seekpoint), pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + /* We need to read seekpoint by seekpoint and do some processing. */ + for (iSeekpoint = 0; iSeekpoint < seekpointCount; ++iSeekpoint) { + drflac_seekpoint* pSeekpoint = (drflac_seekpoint*)pRawData + iSeekpoint; + + if (onRead(pUserData, pSeekpoint, DRFLAC_SEEKPOINT_SIZE_IN_BYTES) != DRFLAC_SEEKPOINT_SIZE_IN_BYTES) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + /* Endian swap. */ + pSeekpoint->firstPCMFrame = drflac__be2host_64(pSeekpoint->firstPCMFrame); + pSeekpoint->flacFrameOffset = drflac__be2host_64(pSeekpoint->flacFrameOffset); + pSeekpoint->pcmFrameCount = drflac__be2host_16(pSeekpoint->pcmFrameCount); + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + metadata.data.seektable.seekpointCount = seekpointCount; + metadata.data.seektable.pSeekpoints = (const drflac_seekpoint*)pRawData; + + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT: + { + if (blockSize < 8) { + return DRFLAC_FALSE; + } + + if (onMeta) { + void* pRawData; + const char* pRunningData; + const char* pRunningDataEnd; + drflac_uint32 i; + + pRawData = drflac__malloc_from_callbacks(blockSize, pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + if (onRead(pUserData, pRawData, blockSize) != blockSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + + pRunningData = (const char*)pRawData; + pRunningDataEnd = (const char*)pRawData + blockSize; + + metadata.data.vorbis_comment.vendorLength = drflac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + + /* Need space for the rest of the block */ + if ((pRunningDataEnd - pRunningData) - 4 < (drflac_int64)metadata.data.vorbis_comment.vendorLength) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; + metadata.data.vorbis_comment.commentCount = drflac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + + /* Need space for 'commentCount' comments after the block, which at minimum is a drflac_uint32 per comment */ + if ((pRunningDataEnd - pRunningData) / sizeof(drflac_uint32) < metadata.data.vorbis_comment.commentCount) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + metadata.data.vorbis_comment.pComments = pRunningData; + + /* Check that the comments section is valid before passing it to the callback */ + for (i = 0; i < metadata.data.vorbis_comment.commentCount; ++i) { + drflac_uint32 commentLength; + + if (pRunningDataEnd - pRunningData < 4) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + commentLength = drflac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + if (pRunningDataEnd - pRunningData < (drflac_int64)commentLength) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + pRunningData += commentLength; + } + + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_CUESHEET: + { + if (blockSize < 396) { + return DRFLAC_FALSE; + } + + if (onMeta) { + void* pRawData; + const char* pRunningData; + const char* pRunningDataEnd; + size_t bufferSize; + drflac_uint8 iTrack; + drflac_uint8 iIndex; + void* pTrackData; + + /* + This needs to be loaded in two passes. The first pass is used to calculate the size of the memory allocation + we need for storing the necessary data. The second pass will fill that buffer with usable data. + */ + pRawData = drflac__malloc_from_callbacks(blockSize, pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + if (onRead(pUserData, pRawData, blockSize) != blockSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + + pRunningData = (const char*)pRawData; + pRunningDataEnd = (const char*)pRawData + blockSize; + + DRFLAC_COPY_MEMORY(metadata.data.cuesheet.catalog, pRunningData, 128); pRunningData += 128; + metadata.data.cuesheet.leadInSampleCount = drflac__be2host_64(*(const drflac_uint64*)pRunningData); pRunningData += 8; + metadata.data.cuesheet.isCD = (pRunningData[0] & 0x80) != 0; pRunningData += 259; + metadata.data.cuesheet.trackCount = pRunningData[0]; pRunningData += 1; + metadata.data.cuesheet.pTrackData = NULL; /* Will be filled later. */ + + /* Pass 1: Calculate the size of the buffer for the track data. */ + { + const char* pRunningDataSaved = pRunningData; /* Will be restored at the end in preparation for the second pass. */ + + bufferSize = metadata.data.cuesheet.trackCount * DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES; + + for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) { + drflac_uint8 indexCount; + drflac_uint32 indexPointSize; + + if (pRunningDataEnd - pRunningData < DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + /* Skip to the index point count */ + pRunningData += 35; + + indexCount = pRunningData[0]; + pRunningData += 1; + + bufferSize += indexCount * sizeof(drflac_cuesheet_track_index); + + /* Quick validation check. */ + indexPointSize = indexCount * DRFLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES; + if (pRunningDataEnd - pRunningData < (drflac_int64)indexPointSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + pRunningData += indexPointSize; + } + + pRunningData = pRunningDataSaved; + } + + /* Pass 2: Allocate a buffer and fill the data. Validation was done in the step above so can be skipped. */ + { + char* pRunningTrackData; + + pTrackData = drflac__malloc_from_callbacks(bufferSize, pAllocationCallbacks); + if (pTrackData == NULL) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + pRunningTrackData = (char*)pTrackData; + + for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) { + drflac_uint8 indexCount; + + DRFLAC_COPY_MEMORY(pRunningTrackData, pRunningData, DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES); + pRunningData += DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES-1; /* Skip forward, but not beyond the last byte in the CUESHEET_TRACK block which is the index count. */ + pRunningTrackData += DRFLAC_CUESHEET_TRACK_SIZE_IN_BYTES-1; + + /* Grab the index count for the next part. */ + indexCount = pRunningData[0]; + pRunningData += 1; + pRunningTrackData += 1; + + /* Extract each track index. */ + for (iIndex = 0; iIndex < indexCount; ++iIndex) { + drflac_cuesheet_track_index* pTrackIndex = (drflac_cuesheet_track_index*)pRunningTrackData; + + DRFLAC_COPY_MEMORY(pRunningTrackData, pRunningData, DRFLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES); + pRunningData += DRFLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES; + pRunningTrackData += sizeof(drflac_cuesheet_track_index); + + pTrackIndex->offset = drflac__be2host_64(pTrackIndex->offset); + } + } + + metadata.data.cuesheet.pTrackData = pTrackData; + } + + /* The original data is no longer needed. */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + pRawData = NULL; + + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pTrackData, pAllocationCallbacks); + pTrackData = NULL; + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_PICTURE: + { + if (blockSize < 32) { + return DRFLAC_FALSE; + } + + if (onMeta) { + void* pRawData; + const char* pRunningData; + const char* pRunningDataEnd; + + pRawData = drflac__malloc_from_callbacks(blockSize, pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + if (onRead(pUserData, pRawData, blockSize) != blockSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + + pRunningData = (const char*)pRawData; + pRunningDataEnd = (const char*)pRawData + blockSize; + + metadata.data.picture.type = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.mimeLength = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + + /* Need space for the rest of the block */ + if ((pRunningDataEnd - pRunningData) - 24 < (drflac_int64)metadata.data.picture.mimeLength) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + metadata.data.picture.mime = pRunningData; pRunningData += metadata.data.picture.mimeLength; + metadata.data.picture.descriptionLength = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + + /* Need space for the rest of the block */ + if ((pRunningDataEnd - pRunningData) - 20 < (drflac_int64)metadata.data.picture.descriptionLength) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + metadata.data.picture.description = pRunningData; pRunningData += metadata.data.picture.descriptionLength; + metadata.data.picture.width = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.height = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.colorDepth = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.indexColorCount = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.pictureDataSize = drflac__be2host_32_ptr_unaligned(pRunningData); pRunningData += 4; + metadata.data.picture.pPictureData = (const drflac_uint8*)pRunningData; + + /* Need space for the picture after the block */ + if (pRunningDataEnd - pRunningData < (drflac_int64)metadata.data.picture.pictureDataSize) { /* <-- Note the order of operations to avoid overflow to a valid value */ + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_PADDING: + { + if (onMeta) { + metadata.data.padding.unused = 0; + + /* Padding doesn't have anything meaningful in it, so just skip over it, but make sure the caller is aware of it by firing the callback. */ + if (!onSeek(pUserData, blockSize, drflac_seek_origin_current)) { + isLastBlock = DRFLAC_TRUE; /* An error occurred while seeking. Attempt to recover by treating this as the last block which will in turn terminate the loop. */ + } else { + onMeta(pUserDataMD, &metadata); + } + } + } break; + + case DRFLAC_METADATA_BLOCK_TYPE_INVALID: + { + /* Invalid chunk. Just skip over this one. */ + if (onMeta) { + if (!onSeek(pUserData, blockSize, drflac_seek_origin_current)) { + isLastBlock = DRFLAC_TRUE; /* An error occurred while seeking. Attempt to recover by treating this as the last block which will in turn terminate the loop. */ + } + } + } break; + + default: + { + /* + It's an unknown chunk, but not necessarily invalid. There's a chance more metadata blocks might be defined later on, so we + can at the very least report the chunk to the application and let it look at the raw data. + */ + if (onMeta) { + void* pRawData = drflac__malloc_from_callbacks(blockSize, pAllocationCallbacks); + if (pRawData == NULL) { + return DRFLAC_FALSE; + } + + if (onRead(pUserData, pRawData, blockSize) != blockSize) { + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + return DRFLAC_FALSE; + } + + metadata.pRawData = pRawData; + metadata.rawDataSize = blockSize; + onMeta(pUserDataMD, &metadata); + + drflac__free_from_callbacks(pRawData, pAllocationCallbacks); + } + } break; + } + + /* If we're not handling metadata, just skip over the block. If we are, it will have been handled earlier in the switch statement above. */ + if (onMeta == NULL && blockSize > 0) { + if (!onSeek(pUserData, blockSize, drflac_seek_origin_current)) { + isLastBlock = DRFLAC_TRUE; + } + } + + runningFilePos += blockSize; + if (isLastBlock) { + break; + } + } + + *pSeektablePos = seektablePos; + *pSeekpointCount = seektableSize / DRFLAC_SEEKPOINT_SIZE_IN_BYTES; + *pFirstFramePos = runningFilePos; + + return DRFLAC_TRUE; +} + +static drflac_bool32 drflac__init_private__native(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, void* pUserDataMD, drflac_bool32 relaxed) +{ + /* Pre Condition: The bit stream should be sitting just past the 4-byte id header. */ + + drflac_uint8 isLastBlock; + drflac_uint8 blockType; + drflac_uint32 blockSize; + + (void)onSeek; + + pInit->container = drflac_container_native; + + /* The first metadata block should be the STREAMINFO block. */ + if (!drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) { + return DRFLAC_FALSE; + } + + if (blockType != DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) { + if (!relaxed) { + /* We're opening in strict mode and the first block is not the STREAMINFO block. Error. */ + return DRFLAC_FALSE; + } else { + /* + Relaxed mode. To open from here we need to just find the first frame and set the sample rate, etc. to whatever is defined + for that frame. + */ + pInit->hasStreamInfoBlock = DRFLAC_FALSE; + pInit->hasMetadataBlocks = DRFLAC_FALSE; + + if (!drflac__read_next_flac_frame_header(&pInit->bs, 0, &pInit->firstFrameHeader)) { + return DRFLAC_FALSE; /* Couldn't find a frame. */ + } + + if (pInit->firstFrameHeader.bitsPerSample == 0) { + return DRFLAC_FALSE; /* Failed to initialize because the first frame depends on the STREAMINFO block, which does not exist. */ + } + + pInit->sampleRate = pInit->firstFrameHeader.sampleRate; + pInit->channels = drflac__get_channel_count_from_channel_assignment(pInit->firstFrameHeader.channelAssignment); + pInit->bitsPerSample = pInit->firstFrameHeader.bitsPerSample; + pInit->maxBlockSizeInPCMFrames = 65535; /* <-- See notes here: https://xiph.org/flac/format.html#metadata_block_streaminfo */ + return DRFLAC_TRUE; + } + } else { + drflac_streaminfo streaminfo; + if (!drflac__read_streaminfo(onRead, pUserData, &streaminfo)) { + return DRFLAC_FALSE; + } + + pInit->hasStreamInfoBlock = DRFLAC_TRUE; + pInit->sampleRate = streaminfo.sampleRate; + pInit->channels = streaminfo.channels; + pInit->bitsPerSample = streaminfo.bitsPerSample; + pInit->totalPCMFrameCount = streaminfo.totalPCMFrameCount; + pInit->maxBlockSizeInPCMFrames = streaminfo.maxBlockSizeInPCMFrames; /* Don't care about the min block size - only the max (used for determining the size of the memory allocation). */ + pInit->hasMetadataBlocks = !isLastBlock; + + if (onMeta) { + drflac_metadata metadata; + metadata.type = DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO; + metadata.pRawData = NULL; + metadata.rawDataSize = 0; + metadata.data.streaminfo = streaminfo; + onMeta(pUserDataMD, &metadata); + } + + return DRFLAC_TRUE; + } +} + +#ifndef DR_FLAC_NO_OGG +#define DRFLAC_OGG_MAX_PAGE_SIZE 65307 +#define DRFLAC_OGG_CAPTURE_PATTERN_CRC32 1605413199 /* CRC-32 of "OggS". */ + +typedef enum +{ + drflac_ogg_recover_on_crc_mismatch, + drflac_ogg_fail_on_crc_mismatch +} drflac_ogg_crc_mismatch_recovery; + +#ifndef DR_FLAC_NO_CRC +static drflac_uint32 drflac__crc32_table[] = { + 0x00000000L, 0x04C11DB7L, 0x09823B6EL, 0x0D4326D9L, + 0x130476DCL, 0x17C56B6BL, 0x1A864DB2L, 0x1E475005L, + 0x2608EDB8L, 0x22C9F00FL, 0x2F8AD6D6L, 0x2B4BCB61L, + 0x350C9B64L, 0x31CD86D3L, 0x3C8EA00AL, 0x384FBDBDL, + 0x4C11DB70L, 0x48D0C6C7L, 0x4593E01EL, 0x4152FDA9L, + 0x5F15ADACL, 0x5BD4B01BL, 0x569796C2L, 0x52568B75L, + 0x6A1936C8L, 0x6ED82B7FL, 0x639B0DA6L, 0x675A1011L, + 0x791D4014L, 0x7DDC5DA3L, 0x709F7B7AL, 0x745E66CDL, + 0x9823B6E0L, 0x9CE2AB57L, 0x91A18D8EL, 0x95609039L, + 0x8B27C03CL, 0x8FE6DD8BL, 0x82A5FB52L, 0x8664E6E5L, + 0xBE2B5B58L, 0xBAEA46EFL, 0xB7A96036L, 0xB3687D81L, + 0xAD2F2D84L, 0xA9EE3033L, 0xA4AD16EAL, 0xA06C0B5DL, + 0xD4326D90L, 0xD0F37027L, 0xDDB056FEL, 0xD9714B49L, + 0xC7361B4CL, 0xC3F706FBL, 0xCEB42022L, 0xCA753D95L, + 0xF23A8028L, 0xF6FB9D9FL, 0xFBB8BB46L, 0xFF79A6F1L, + 0xE13EF6F4L, 0xE5FFEB43L, 0xE8BCCD9AL, 0xEC7DD02DL, + 0x34867077L, 0x30476DC0L, 0x3D044B19L, 0x39C556AEL, + 0x278206ABL, 0x23431B1CL, 0x2E003DC5L, 0x2AC12072L, + 0x128E9DCFL, 0x164F8078L, 0x1B0CA6A1L, 0x1FCDBB16L, + 0x018AEB13L, 0x054BF6A4L, 0x0808D07DL, 0x0CC9CDCAL, + 0x7897AB07L, 0x7C56B6B0L, 0x71159069L, 0x75D48DDEL, + 0x6B93DDDBL, 0x6F52C06CL, 0x6211E6B5L, 0x66D0FB02L, + 0x5E9F46BFL, 0x5A5E5B08L, 0x571D7DD1L, 0x53DC6066L, + 0x4D9B3063L, 0x495A2DD4L, 0x44190B0DL, 0x40D816BAL, + 0xACA5C697L, 0xA864DB20L, 0xA527FDF9L, 0xA1E6E04EL, + 0xBFA1B04BL, 0xBB60ADFCL, 0xB6238B25L, 0xB2E29692L, + 0x8AAD2B2FL, 0x8E6C3698L, 0x832F1041L, 0x87EE0DF6L, + 0x99A95DF3L, 0x9D684044L, 0x902B669DL, 0x94EA7B2AL, + 0xE0B41DE7L, 0xE4750050L, 0xE9362689L, 0xEDF73B3EL, + 0xF3B06B3BL, 0xF771768CL, 0xFA325055L, 0xFEF34DE2L, + 0xC6BCF05FL, 0xC27DEDE8L, 0xCF3ECB31L, 0xCBFFD686L, + 0xD5B88683L, 0xD1799B34L, 0xDC3ABDEDL, 0xD8FBA05AL, + 0x690CE0EEL, 0x6DCDFD59L, 0x608EDB80L, 0x644FC637L, + 0x7A089632L, 0x7EC98B85L, 0x738AAD5CL, 0x774BB0EBL, + 0x4F040D56L, 0x4BC510E1L, 0x46863638L, 0x42472B8FL, + 0x5C007B8AL, 0x58C1663DL, 0x558240E4L, 0x51435D53L, + 0x251D3B9EL, 0x21DC2629L, 0x2C9F00F0L, 0x285E1D47L, + 0x36194D42L, 0x32D850F5L, 0x3F9B762CL, 0x3B5A6B9BL, + 0x0315D626L, 0x07D4CB91L, 0x0A97ED48L, 0x0E56F0FFL, + 0x1011A0FAL, 0x14D0BD4DL, 0x19939B94L, 0x1D528623L, + 0xF12F560EL, 0xF5EE4BB9L, 0xF8AD6D60L, 0xFC6C70D7L, + 0xE22B20D2L, 0xE6EA3D65L, 0xEBA91BBCL, 0xEF68060BL, + 0xD727BBB6L, 0xD3E6A601L, 0xDEA580D8L, 0xDA649D6FL, + 0xC423CD6AL, 0xC0E2D0DDL, 0xCDA1F604L, 0xC960EBB3L, + 0xBD3E8D7EL, 0xB9FF90C9L, 0xB4BCB610L, 0xB07DABA7L, + 0xAE3AFBA2L, 0xAAFBE615L, 0xA7B8C0CCL, 0xA379DD7BL, + 0x9B3660C6L, 0x9FF77D71L, 0x92B45BA8L, 0x9675461FL, + 0x8832161AL, 0x8CF30BADL, 0x81B02D74L, 0x857130C3L, + 0x5D8A9099L, 0x594B8D2EL, 0x5408ABF7L, 0x50C9B640L, + 0x4E8EE645L, 0x4A4FFBF2L, 0x470CDD2BL, 0x43CDC09CL, + 0x7B827D21L, 0x7F436096L, 0x7200464FL, 0x76C15BF8L, + 0x68860BFDL, 0x6C47164AL, 0x61043093L, 0x65C52D24L, + 0x119B4BE9L, 0x155A565EL, 0x18197087L, 0x1CD86D30L, + 0x029F3D35L, 0x065E2082L, 0x0B1D065BL, 0x0FDC1BECL, + 0x3793A651L, 0x3352BBE6L, 0x3E119D3FL, 0x3AD08088L, + 0x2497D08DL, 0x2056CD3AL, 0x2D15EBE3L, 0x29D4F654L, + 0xC5A92679L, 0xC1683BCEL, 0xCC2B1D17L, 0xC8EA00A0L, + 0xD6AD50A5L, 0xD26C4D12L, 0xDF2F6BCBL, 0xDBEE767CL, + 0xE3A1CBC1L, 0xE760D676L, 0xEA23F0AFL, 0xEEE2ED18L, + 0xF0A5BD1DL, 0xF464A0AAL, 0xF9278673L, 0xFDE69BC4L, + 0x89B8FD09L, 0x8D79E0BEL, 0x803AC667L, 0x84FBDBD0L, + 0x9ABC8BD5L, 0x9E7D9662L, 0x933EB0BBL, 0x97FFAD0CL, + 0xAFB010B1L, 0xAB710D06L, 0xA6322BDFL, 0xA2F33668L, + 0xBCB4666DL, 0xB8757BDAL, 0xB5365D03L, 0xB1F740B4L +}; +#endif + +static DRFLAC_INLINE drflac_uint32 drflac_crc32_byte(drflac_uint32 crc32, drflac_uint8 data) +{ +#ifndef DR_FLAC_NO_CRC + return (crc32 << 8) ^ drflac__crc32_table[(drflac_uint8)((crc32 >> 24) & 0xFF) ^ data]; +#else + (void)data; + return crc32; +#endif +} + +#if 0 +static DRFLAC_INLINE drflac_uint32 drflac_crc32_uint32(drflac_uint32 crc32, drflac_uint32 data) +{ + crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 24) & 0xFF)); + crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 16) & 0xFF)); + crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 8) & 0xFF)); + crc32 = drflac_crc32_byte(crc32, (drflac_uint8)((data >> 0) & 0xFF)); + return crc32; +} + +static DRFLAC_INLINE drflac_uint32 drflac_crc32_uint64(drflac_uint32 crc32, drflac_uint64 data) +{ + crc32 = drflac_crc32_uint32(crc32, (drflac_uint32)((data >> 32) & 0xFFFFFFFF)); + crc32 = drflac_crc32_uint32(crc32, (drflac_uint32)((data >> 0) & 0xFFFFFFFF)); + return crc32; +} +#endif + +static DRFLAC_INLINE drflac_uint32 drflac_crc32_buffer(drflac_uint32 crc32, drflac_uint8* pData, drflac_uint32 dataSize) +{ + /* This can be optimized. */ + drflac_uint32 i; + for (i = 0; i < dataSize; ++i) { + crc32 = drflac_crc32_byte(crc32, pData[i]); + } + return crc32; +} + + +static DRFLAC_INLINE drflac_bool32 drflac_ogg__is_capture_pattern(drflac_uint8 pattern[4]) +{ + return pattern[0] == 'O' && pattern[1] == 'g' && pattern[2] == 'g' && pattern[3] == 'S'; +} + +static DRFLAC_INLINE drflac_uint32 drflac_ogg__get_page_header_size(drflac_ogg_page_header* pHeader) +{ + return 27 + pHeader->segmentCount; +} + +static DRFLAC_INLINE drflac_uint32 drflac_ogg__get_page_body_size(drflac_ogg_page_header* pHeader) +{ + drflac_uint32 pageBodySize = 0; + int i; + + for (i = 0; i < pHeader->segmentCount; ++i) { + pageBodySize += pHeader->segmentTable[i]; + } + + return pageBodySize; +} + +static drflac_result drflac_ogg__read_page_header_after_capture_pattern(drflac_read_proc onRead, void* pUserData, drflac_ogg_page_header* pHeader, drflac_uint32* pBytesRead, drflac_uint32* pCRC32) +{ + drflac_uint8 data[23]; + drflac_uint32 i; + + DRFLAC_ASSERT(*pCRC32 == DRFLAC_OGG_CAPTURE_PATTERN_CRC32); + + if (onRead(pUserData, data, 23) != 23) { + return DRFLAC_AT_END; + } + *pBytesRead += 23; + + /* + It's not actually used, but set the capture pattern to 'OggS' for completeness. Not doing this will cause static analysers to complain about + us trying to access uninitialized data. We could alternatively just comment out this member of the drflac_ogg_page_header structure, but I + like to have it map to the structure of the underlying data. + */ + pHeader->capturePattern[0] = 'O'; + pHeader->capturePattern[1] = 'g'; + pHeader->capturePattern[2] = 'g'; + pHeader->capturePattern[3] = 'S'; + + pHeader->structureVersion = data[0]; + pHeader->headerType = data[1]; + DRFLAC_COPY_MEMORY(&pHeader->granulePosition, &data[ 2], 8); + DRFLAC_COPY_MEMORY(&pHeader->serialNumber, &data[10], 4); + DRFLAC_COPY_MEMORY(&pHeader->sequenceNumber, &data[14], 4); + DRFLAC_COPY_MEMORY(&pHeader->checksum, &data[18], 4); + pHeader->segmentCount = data[22]; + + /* Calculate the CRC. Note that for the calculation the checksum part of the page needs to be set to 0. */ + data[18] = 0; + data[19] = 0; + data[20] = 0; + data[21] = 0; + + for (i = 0; i < 23; ++i) { + *pCRC32 = drflac_crc32_byte(*pCRC32, data[i]); + } + + + if (onRead(pUserData, pHeader->segmentTable, pHeader->segmentCount) != pHeader->segmentCount) { + return DRFLAC_AT_END; + } + *pBytesRead += pHeader->segmentCount; + + for (i = 0; i < pHeader->segmentCount; ++i) { + *pCRC32 = drflac_crc32_byte(*pCRC32, pHeader->segmentTable[i]); + } + + return DRFLAC_SUCCESS; +} + +static drflac_result drflac_ogg__read_page_header(drflac_read_proc onRead, void* pUserData, drflac_ogg_page_header* pHeader, drflac_uint32* pBytesRead, drflac_uint32* pCRC32) +{ + drflac_uint8 id[4]; + + *pBytesRead = 0; + + if (onRead(pUserData, id, 4) != 4) { + return DRFLAC_AT_END; + } + *pBytesRead += 4; + + /* We need to read byte-by-byte until we find the OggS capture pattern. */ + for (;;) { + if (drflac_ogg__is_capture_pattern(id)) { + drflac_result result; + + *pCRC32 = DRFLAC_OGG_CAPTURE_PATTERN_CRC32; + + result = drflac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, pHeader, pBytesRead, pCRC32); + if (result == DRFLAC_SUCCESS) { + return DRFLAC_SUCCESS; + } else { + if (result == DRFLAC_CRC_MISMATCH) { + continue; + } else { + return result; + } + } + } else { + /* The first 4 bytes did not equal the capture pattern. Read the next byte and try again. */ + id[0] = id[1]; + id[1] = id[2]; + id[2] = id[3]; + if (onRead(pUserData, &id[3], 1) != 1) { + return DRFLAC_AT_END; + } + *pBytesRead += 1; + } + } +} + + +/* +The main part of the Ogg encapsulation is the conversion from the physical Ogg bitstream to the native FLAC bitstream. It works +in three general stages: Ogg Physical Bitstream -> Ogg/FLAC Logical Bitstream -> FLAC Native Bitstream. dr_flac is designed +in such a way that the core sections assume everything is delivered in native format. Therefore, for each encapsulation type +dr_flac is supporting there needs to be a layer sitting on top of the onRead and onSeek callbacks that ensures the bits read from +the physical Ogg bitstream are converted and delivered in native FLAC format. +*/ +typedef struct +{ + drflac_read_proc onRead; /* The original onRead callback from drflac_open() and family. */ + drflac_seek_proc onSeek; /* The original onSeek callback from drflac_open() and family. */ + void* pUserData; /* The user data passed on onRead and onSeek. This is the user data that was passed on drflac_open() and family. */ + drflac_uint64 currentBytePos; /* The position of the byte we are sitting on in the physical byte stream. Used for efficient seeking. */ + drflac_uint64 firstBytePos; /* The position of the first byte in the physical bitstream. Points to the start of the "OggS" identifier of the FLAC bos page. */ + drflac_uint32 serialNumber; /* The serial number of the FLAC audio pages. This is determined by the initial header page that was read during initialization. */ + drflac_ogg_page_header bosPageHeader; /* Used for seeking. */ + drflac_ogg_page_header currentPageHeader; + drflac_uint32 bytesRemainingInPage; + drflac_uint32 pageDataSize; + drflac_uint8 pageData[DRFLAC_OGG_MAX_PAGE_SIZE]; +} drflac_oggbs; /* oggbs = Ogg Bitstream */ + +static size_t drflac_oggbs__read_physical(drflac_oggbs* oggbs, void* bufferOut, size_t bytesToRead) +{ + size_t bytesActuallyRead = oggbs->onRead(oggbs->pUserData, bufferOut, bytesToRead); + oggbs->currentBytePos += bytesActuallyRead; + + return bytesActuallyRead; +} + +static drflac_bool32 drflac_oggbs__seek_physical(drflac_oggbs* oggbs, drflac_uint64 offset, drflac_seek_origin origin) +{ + if (origin == drflac_seek_origin_start) { + if (offset <= 0x7FFFFFFF) { + if (!oggbs->onSeek(oggbs->pUserData, (int)offset, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + oggbs->currentBytePos = offset; + + return DRFLAC_TRUE; + } else { + if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + oggbs->currentBytePos = offset; + + return drflac_oggbs__seek_physical(oggbs, offset - 0x7FFFFFFF, drflac_seek_origin_current); + } + } else { + while (offset > 0x7FFFFFFF) { + if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + oggbs->currentBytePos += 0x7FFFFFFF; + offset -= 0x7FFFFFFF; + } + + if (!oggbs->onSeek(oggbs->pUserData, (int)offset, drflac_seek_origin_current)) { /* <-- Safe cast thanks to the loop above. */ + return DRFLAC_FALSE; + } + oggbs->currentBytePos += offset; + + return DRFLAC_TRUE; + } +} + +static drflac_bool32 drflac_oggbs__goto_next_page(drflac_oggbs* oggbs, drflac_ogg_crc_mismatch_recovery recoveryMethod) +{ + drflac_ogg_page_header header; + for (;;) { + drflac_uint32 crc32 = 0; + drflac_uint32 bytesRead; + drflac_uint32 pageBodySize; +#ifndef DR_FLAC_NO_CRC + drflac_uint32 actualCRC32; +#endif + + if (drflac_ogg__read_page_header(oggbs->onRead, oggbs->pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) { + return DRFLAC_FALSE; + } + oggbs->currentBytePos += bytesRead; + + pageBodySize = drflac_ogg__get_page_body_size(&header); + if (pageBodySize > DRFLAC_OGG_MAX_PAGE_SIZE) { + continue; /* Invalid page size. Assume it's corrupted and just move to the next page. */ + } + + if (header.serialNumber != oggbs->serialNumber) { + /* It's not a FLAC page. Skip it. */ + if (pageBodySize > 0 && !drflac_oggbs__seek_physical(oggbs, pageBodySize, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + continue; + } + + + /* We need to read the entire page and then do a CRC check on it. If there's a CRC mismatch we need to skip this page. */ + if (drflac_oggbs__read_physical(oggbs, oggbs->pageData, pageBodySize) != pageBodySize) { + return DRFLAC_FALSE; + } + oggbs->pageDataSize = pageBodySize; + +#ifndef DR_FLAC_NO_CRC + actualCRC32 = drflac_crc32_buffer(crc32, oggbs->pageData, oggbs->pageDataSize); + if (actualCRC32 != header.checksum) { + if (recoveryMethod == drflac_ogg_recover_on_crc_mismatch) { + continue; /* CRC mismatch. Skip this page. */ + } else { + /* + Even though we are failing on a CRC mismatch, we still want our stream to be in a good state. Therefore we + go to the next valid page to ensure we're in a good state, but return false to let the caller know that the + seek did not fully complete. + */ + drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch); + return DRFLAC_FALSE; + } + } +#else + (void)recoveryMethod; /* <-- Silence a warning. */ +#endif + + oggbs->currentPageHeader = header; + oggbs->bytesRemainingInPage = pageBodySize; + return DRFLAC_TRUE; + } +} + +/* Function below is unused at the moment, but I might be re-adding it later. */ +#if 0 +static drflac_uint8 drflac_oggbs__get_current_segment_index(drflac_oggbs* oggbs, drflac_uint8* pBytesRemainingInSeg) +{ + drflac_uint32 bytesConsumedInPage = drflac_ogg__get_page_body_size(&oggbs->currentPageHeader) - oggbs->bytesRemainingInPage; + drflac_uint8 iSeg = 0; + drflac_uint32 iByte = 0; + while (iByte < bytesConsumedInPage) { + drflac_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg]; + if (iByte + segmentSize > bytesConsumedInPage) { + break; + } else { + iSeg += 1; + iByte += segmentSize; + } + } + + *pBytesRemainingInSeg = oggbs->currentPageHeader.segmentTable[iSeg] - (drflac_uint8)(bytesConsumedInPage - iByte); + return iSeg; +} + +static drflac_bool32 drflac_oggbs__seek_to_next_packet(drflac_oggbs* oggbs) +{ + /* The current packet ends when we get to the segment with a lacing value of < 255 which is not at the end of a page. */ + for (;;) { + drflac_bool32 atEndOfPage = DRFLAC_FALSE; + + drflac_uint8 bytesRemainingInSeg; + drflac_uint8 iFirstSeg = drflac_oggbs__get_current_segment_index(oggbs, &bytesRemainingInSeg); + + drflac_uint32 bytesToEndOfPacketOrPage = bytesRemainingInSeg; + for (drflac_uint8 iSeg = iFirstSeg; iSeg < oggbs->currentPageHeader.segmentCount; ++iSeg) { + drflac_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg]; + if (segmentSize < 255) { + if (iSeg == oggbs->currentPageHeader.segmentCount-1) { + atEndOfPage = DRFLAC_TRUE; + } + + break; + } + + bytesToEndOfPacketOrPage += segmentSize; + } + + /* + At this point we will have found either the packet or the end of the page. If were at the end of the page we'll + want to load the next page and keep searching for the end of the packet. + */ + drflac_oggbs__seek_physical(oggbs, bytesToEndOfPacketOrPage, drflac_seek_origin_current); + oggbs->bytesRemainingInPage -= bytesToEndOfPacketOrPage; + + if (atEndOfPage) { + /* + We're potentially at the next packet, but we need to check the next page first to be sure because the packet may + straddle pages. + */ + if (!drflac_oggbs__goto_next_page(oggbs)) { + return DRFLAC_FALSE; + } + + /* If it's a fresh packet it most likely means we're at the next packet. */ + if ((oggbs->currentPageHeader.headerType & 0x01) == 0) { + return DRFLAC_TRUE; + } + } else { + /* We're at the next packet. */ + return DRFLAC_TRUE; + } + } +} + +static drflac_bool32 drflac_oggbs__seek_to_next_frame(drflac_oggbs* oggbs) +{ + /* The bitstream should be sitting on the first byte just after the header of the frame. */ + + /* What we're actually doing here is seeking to the start of the next packet. */ + return drflac_oggbs__seek_to_next_packet(oggbs); +} +#endif + +static size_t drflac__on_read_ogg(void* pUserData, void* bufferOut, size_t bytesToRead) +{ + drflac_oggbs* oggbs = (drflac_oggbs*)pUserData; + drflac_uint8* pRunningBufferOut = (drflac_uint8*)bufferOut; + size_t bytesRead = 0; + + DRFLAC_ASSERT(oggbs != NULL); + DRFLAC_ASSERT(pRunningBufferOut != NULL); + + /* Reading is done page-by-page. If we've run out of bytes in the page we need to move to the next one. */ + while (bytesRead < bytesToRead) { + size_t bytesRemainingToRead = bytesToRead - bytesRead; + + if (oggbs->bytesRemainingInPage >= bytesRemainingToRead) { + DRFLAC_COPY_MEMORY(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), bytesRemainingToRead); + bytesRead += bytesRemainingToRead; + oggbs->bytesRemainingInPage -= (drflac_uint32)bytesRemainingToRead; + break; + } + + /* If we get here it means some of the requested data is contained in the next pages. */ + if (oggbs->bytesRemainingInPage > 0) { + DRFLAC_COPY_MEMORY(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), oggbs->bytesRemainingInPage); + bytesRead += oggbs->bytesRemainingInPage; + pRunningBufferOut += oggbs->bytesRemainingInPage; + oggbs->bytesRemainingInPage = 0; + } + + DRFLAC_ASSERT(bytesRemainingToRead > 0); + if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) { + break; /* Failed to go to the next page. Might have simply hit the end of the stream. */ + } + } + + return bytesRead; +} + +static drflac_bool32 drflac__on_seek_ogg(void* pUserData, int offset, drflac_seek_origin origin) +{ + drflac_oggbs* oggbs = (drflac_oggbs*)pUserData; + int bytesSeeked = 0; + + DRFLAC_ASSERT(oggbs != NULL); + DRFLAC_ASSERT(offset >= 0); /* <-- Never seek backwards. */ + + /* Seeking is always forward which makes things a lot simpler. */ + if (origin == drflac_seek_origin_start) { + if (!drflac_oggbs__seek_physical(oggbs, (int)oggbs->firstBytePos, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + + if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_fail_on_crc_mismatch)) { + return DRFLAC_FALSE; + } + + return drflac__on_seek_ogg(pUserData, offset, drflac_seek_origin_current); + } + + DRFLAC_ASSERT(origin == drflac_seek_origin_current); + + while (bytesSeeked < offset) { + int bytesRemainingToSeek = offset - bytesSeeked; + DRFLAC_ASSERT(bytesRemainingToSeek >= 0); + + if (oggbs->bytesRemainingInPage >= (size_t)bytesRemainingToSeek) { + bytesSeeked += bytesRemainingToSeek; + (void)bytesSeeked; /* <-- Silence a dead store warning emitted by Clang Static Analyzer. */ + oggbs->bytesRemainingInPage -= bytesRemainingToSeek; + break; + } + + /* If we get here it means some of the requested data is contained in the next pages. */ + if (oggbs->bytesRemainingInPage > 0) { + bytesSeeked += (int)oggbs->bytesRemainingInPage; + oggbs->bytesRemainingInPage = 0; + } + + DRFLAC_ASSERT(bytesRemainingToSeek > 0); + if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_fail_on_crc_mismatch)) { + /* Failed to go to the next page. We either hit the end of the stream or had a CRC mismatch. */ + return DRFLAC_FALSE; + } + } + + return DRFLAC_TRUE; +} + + +static drflac_bool32 drflac_ogg__seek_to_pcm_frame(drflac* pFlac, drflac_uint64 pcmFrameIndex) +{ + drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; + drflac_uint64 originalBytePos; + drflac_uint64 runningGranulePosition; + drflac_uint64 runningFrameBytePos; + drflac_uint64 runningPCMFrameCount; + + DRFLAC_ASSERT(oggbs != NULL); + + originalBytePos = oggbs->currentBytePos; /* For recovery. Points to the OggS identifier. */ + + /* First seek to the first frame. */ + if (!drflac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes)) { + return DRFLAC_FALSE; + } + oggbs->bytesRemainingInPage = 0; + + runningGranulePosition = 0; + for (;;) { + if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) { + drflac_oggbs__seek_physical(oggbs, originalBytePos, drflac_seek_origin_start); + return DRFLAC_FALSE; /* Never did find that sample... */ + } + + runningFrameBytePos = oggbs->currentBytePos - drflac_ogg__get_page_header_size(&oggbs->currentPageHeader) - oggbs->pageDataSize; + if (oggbs->currentPageHeader.granulePosition >= pcmFrameIndex) { + break; /* The sample is somewhere in the previous page. */ + } + + /* + At this point we know the sample is not in the previous page. It could possibly be in this page. For simplicity we + disregard any pages that do not begin a fresh packet. + */ + if ((oggbs->currentPageHeader.headerType & 0x01) == 0) { /* <-- Is it a fresh page? */ + if (oggbs->currentPageHeader.segmentTable[0] >= 2) { + drflac_uint8 firstBytesInPage[2]; + firstBytesInPage[0] = oggbs->pageData[0]; + firstBytesInPage[1] = oggbs->pageData[1]; + + if ((firstBytesInPage[0] == 0xFF) && (firstBytesInPage[1] & 0xFC) == 0xF8) { /* <-- Does the page begin with a frame's sync code? */ + runningGranulePosition = oggbs->currentPageHeader.granulePosition; + } + + continue; + } + } + } + + /* + We found the page that that is closest to the sample, so now we need to find it. The first thing to do is seek to the + start of that page. In the loop above we checked that it was a fresh page which means this page is also the start of + a new frame. This property means that after we've seeked to the page we can immediately start looping over frames until + we find the one containing the target sample. + */ + if (!drflac_oggbs__seek_physical(oggbs, runningFrameBytePos, drflac_seek_origin_start)) { + return DRFLAC_FALSE; + } + if (!drflac_oggbs__goto_next_page(oggbs, drflac_ogg_recover_on_crc_mismatch)) { + return DRFLAC_FALSE; + } + + /* + At this point we'll be sitting on the first byte of the frame header of the first frame in the page. We just keep + looping over these frames until we find the one containing the sample we're after. + */ + runningPCMFrameCount = runningGranulePosition; + for (;;) { + /* + There are two ways to find the sample and seek past irrelevant frames: + 1) Use the native FLAC decoder. + 2) Use Ogg's framing system. + + Both of these options have their own pros and cons. Using the native FLAC decoder is slower because it needs to + do a full decode of the frame. Using Ogg's framing system is faster, but more complicated and involves some code + duplication for the decoding of frame headers. + + Another thing to consider is that using the Ogg framing system will perform direct seeking of the physical Ogg + bitstream. This is important to consider because it means we cannot read data from the drflac_bs object using the + standard drflac__*() APIs because that will read in extra data for its own internal caching which in turn breaks + the positioning of the read pointer of the physical Ogg bitstream. Therefore, anything that would normally be read + using the native FLAC decoding APIs, such as drflac__read_next_flac_frame_header(), need to be re-implemented so as to + avoid the use of the drflac_bs object. + + Considering these issues, I have decided to use the slower native FLAC decoding method for the following reasons: + 1) Seeking is already partially accelerated using Ogg's paging system in the code block above. + 2) Seeking in an Ogg encapsulated FLAC stream is probably quite uncommon. + 3) Simplicity. + */ + drflac_uint64 firstPCMFrameInFLACFrame = 0; + drflac_uint64 lastPCMFrameInFLACFrame = 0; + drflac_uint64 pcmFrameCountInThisFrame; + + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return DRFLAC_FALSE; + } + + drflac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame); + + pcmFrameCountInThisFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1; + + /* If we are seeking to the end of the file and we've just hit it, we're done. */ + if (pcmFrameIndex == pFlac->totalPCMFrameCount && (runningPCMFrameCount + pcmFrameCountInThisFrame) == pFlac->totalPCMFrameCount) { + drflac_result result = drflac__decode_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + pFlac->currentPCMFrame = pcmFrameIndex; + pFlac->currentFLACFrame.pcmFramesRemaining = 0; + return DRFLAC_TRUE; + } else { + return DRFLAC_FALSE; + } + } + + if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFrame)) { + /* + The sample should be in this FLAC frame. We need to fully decode it, however if it's an invalid frame (a CRC mismatch), we need to pretend + it never existed and keep iterating. + */ + drflac_result result = drflac__decode_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + /* The frame is valid. We just need to skip over some samples to ensure it's sample-exact. */ + drflac_uint64 pcmFramesToDecode = (size_t)(pcmFrameIndex - runningPCMFrameCount); /* <-- Safe cast because the maximum number of samples in a frame is 65535. */ + if (pcmFramesToDecode == 0) { + return DRFLAC_TRUE; + } + + pFlac->currentPCMFrame = runningPCMFrameCount; + + return drflac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode; /* <-- If this fails, something bad has happened (it should never fail). */ + } else { + if (result == DRFLAC_CRC_MISMATCH) { + continue; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } else { + /* + It's not in this frame. We need to seek past the frame, but check if there was a CRC mismatch. If so, we pretend this + frame never existed and leave the running sample count untouched. + */ + drflac_result result = drflac__seek_to_next_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + runningPCMFrameCount += pcmFrameCountInThisFrame; + } else { + if (result == DRFLAC_CRC_MISMATCH) { + continue; /* CRC mismatch. Pretend this frame never existed. */ + } else { + return DRFLAC_FALSE; + } + } + } + } +} + + + +static drflac_bool32 drflac__init_private__ogg(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, void* pUserDataMD, drflac_bool32 relaxed) +{ + drflac_ogg_page_header header; + drflac_uint32 crc32 = DRFLAC_OGG_CAPTURE_PATTERN_CRC32; + drflac_uint32 bytesRead = 0; + + /* Pre Condition: The bit stream should be sitting just past the 4-byte OggS capture pattern. */ + (void)relaxed; + + pInit->container = drflac_container_ogg; + pInit->oggFirstBytePos = 0; + + /* + We'll get here if the first 4 bytes of the stream were the OggS capture pattern, however it doesn't necessarily mean the + stream includes FLAC encoded audio. To check for this we need to scan the beginning-of-stream page markers and check if + any match the FLAC specification. Important to keep in mind that the stream may be multiplexed. + */ + if (drflac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) { + return DRFLAC_FALSE; + } + pInit->runningFilePos += bytesRead; + + for (;;) { + int pageBodySize; + + /* Break if we're past the beginning of stream page. */ + if ((header.headerType & 0x02) == 0) { + return DRFLAC_FALSE; + } + + /* Check if it's a FLAC header. */ + pageBodySize = drflac_ogg__get_page_body_size(&header); + if (pageBodySize == 51) { /* 51 = the lacing value of the FLAC header packet. */ + /* It could be a FLAC page... */ + drflac_uint32 bytesRemainingInPage = pageBodySize; + drflac_uint8 packetType; + + if (onRead(pUserData, &packetType, 1) != 1) { + return DRFLAC_FALSE; + } + + bytesRemainingInPage -= 1; + if (packetType == 0x7F) { + /* Increasingly more likely to be a FLAC page... */ + drflac_uint8 sig[4]; + if (onRead(pUserData, sig, 4) != 4) { + return DRFLAC_FALSE; + } + + bytesRemainingInPage -= 4; + if (sig[0] == 'F' && sig[1] == 'L' && sig[2] == 'A' && sig[3] == 'C') { + /* Almost certainly a FLAC page... */ + drflac_uint8 mappingVersion[2]; + if (onRead(pUserData, mappingVersion, 2) != 2) { + return DRFLAC_FALSE; + } + + if (mappingVersion[0] != 1) { + return DRFLAC_FALSE; /* Only supporting version 1.x of the Ogg mapping. */ + } + + /* + The next 2 bytes are the non-audio packets, not including this one. We don't care about this because we're going to + be handling it in a generic way based on the serial number and packet types. + */ + if (!onSeek(pUserData, 2, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + + /* Expecting the native FLAC signature "fLaC". */ + if (onRead(pUserData, sig, 4) != 4) { + return DRFLAC_FALSE; + } + + if (sig[0] == 'f' && sig[1] == 'L' && sig[2] == 'a' && sig[3] == 'C') { + /* The remaining data in the page should be the STREAMINFO block. */ + drflac_streaminfo streaminfo; + drflac_uint8 isLastBlock; + drflac_uint8 blockType; + drflac_uint32 blockSize; + if (!drflac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) { + return DRFLAC_FALSE; + } + + if (blockType != DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) { + return DRFLAC_FALSE; /* Invalid block type. First block must be the STREAMINFO block. */ + } + + if (drflac__read_streaminfo(onRead, pUserData, &streaminfo)) { + /* Success! */ + pInit->hasStreamInfoBlock = DRFLAC_TRUE; + pInit->sampleRate = streaminfo.sampleRate; + pInit->channels = streaminfo.channels; + pInit->bitsPerSample = streaminfo.bitsPerSample; + pInit->totalPCMFrameCount = streaminfo.totalPCMFrameCount; + pInit->maxBlockSizeInPCMFrames = streaminfo.maxBlockSizeInPCMFrames; + pInit->hasMetadataBlocks = !isLastBlock; + + if (onMeta) { + drflac_metadata metadata; + metadata.type = DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO; + metadata.pRawData = NULL; + metadata.rawDataSize = 0; + metadata.data.streaminfo = streaminfo; + onMeta(pUserDataMD, &metadata); + } + + pInit->runningFilePos += pageBodySize; + pInit->oggFirstBytePos = pInit->runningFilePos - 79; /* Subtracting 79 will place us right on top of the "OggS" identifier of the FLAC bos page. */ + pInit->oggSerial = header.serialNumber; + pInit->oggBosHeader = header; + break; + } else { + /* Failed to read STREAMINFO block. Aww, so close... */ + return DRFLAC_FALSE; + } + } else { + /* Invalid file. */ + return DRFLAC_FALSE; + } + } else { + /* Not a FLAC header. Skip it. */ + if (!onSeek(pUserData, bytesRemainingInPage, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + } + } else { + /* Not a FLAC header. Seek past the entire page and move on to the next. */ + if (!onSeek(pUserData, bytesRemainingInPage, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + } + } else { + if (!onSeek(pUserData, pageBodySize, drflac_seek_origin_current)) { + return DRFLAC_FALSE; + } + } + + pInit->runningFilePos += pageBodySize; + + + /* Read the header of the next page. */ + if (drflac_ogg__read_page_header(onRead, pUserData, &header, &bytesRead, &crc32) != DRFLAC_SUCCESS) { + return DRFLAC_FALSE; + } + pInit->runningFilePos += bytesRead; + } + + /* + If we get here it means we found a FLAC audio stream. We should be sitting on the first byte of the header of the next page. The next + packets in the FLAC logical stream contain the metadata. The only thing left to do in the initialization phase for Ogg is to create the + Ogg bistream object. + */ + pInit->hasMetadataBlocks = DRFLAC_TRUE; /* <-- Always have at least VORBIS_COMMENT metadata block. */ + return DRFLAC_TRUE; +} +#endif + +static drflac_bool32 drflac__init_private(drflac_init_info* pInit, drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, void* pUserDataMD) +{ + drflac_bool32 relaxed; + drflac_uint8 id[4]; + + if (pInit == NULL || onRead == NULL || onSeek == NULL) { + return DRFLAC_FALSE; + } + + DRFLAC_ZERO_MEMORY(pInit, sizeof(*pInit)); + pInit->onRead = onRead; + pInit->onSeek = onSeek; + pInit->onMeta = onMeta; + pInit->container = container; + pInit->pUserData = pUserData; + pInit->pUserDataMD = pUserDataMD; + + pInit->bs.onRead = onRead; + pInit->bs.onSeek = onSeek; + pInit->bs.pUserData = pUserData; + drflac__reset_cache(&pInit->bs); + + + /* If the container is explicitly defined then we can try opening in relaxed mode. */ + relaxed = container != drflac_container_unknown; + + /* Skip over any ID3 tags. */ + for (;;) { + if (onRead(pUserData, id, 4) != 4) { + return DRFLAC_FALSE; /* Ran out of data. */ + } + pInit->runningFilePos += 4; + + if (id[0] == 'I' && id[1] == 'D' && id[2] == '3') { + drflac_uint8 header[6]; + drflac_uint8 flags; + drflac_uint32 headerSize; + + if (onRead(pUserData, header, 6) != 6) { + return DRFLAC_FALSE; /* Ran out of data. */ + } + pInit->runningFilePos += 6; + + flags = header[1]; + + DRFLAC_COPY_MEMORY(&headerSize, header+2, 4); + headerSize = drflac__unsynchsafe_32(drflac__be2host_32(headerSize)); + if (flags & 0x10) { + headerSize += 10; + } + + if (!onSeek(pUserData, headerSize, drflac_seek_origin_current)) { + return DRFLAC_FALSE; /* Failed to seek past the tag. */ + } + pInit->runningFilePos += headerSize; + } else { + break; + } + } + + if (id[0] == 'f' && id[1] == 'L' && id[2] == 'a' && id[3] == 'C') { + return drflac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); + } +#ifndef DR_FLAC_NO_OGG + if (id[0] == 'O' && id[1] == 'g' && id[2] == 'g' && id[3] == 'S') { + return drflac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); + } +#endif + + /* If we get here it means we likely don't have a header. Try opening in relaxed mode, if applicable. */ + if (relaxed) { + if (container == drflac_container_native) { + return drflac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); + } +#ifndef DR_FLAC_NO_OGG + if (container == drflac_container_ogg) { + return drflac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); + } +#endif + } + + /* Unsupported container. */ + return DRFLAC_FALSE; +} + +static void drflac__init_from_info(drflac* pFlac, const drflac_init_info* pInit) +{ + DRFLAC_ASSERT(pFlac != NULL); + DRFLAC_ASSERT(pInit != NULL); + + DRFLAC_ZERO_MEMORY(pFlac, sizeof(*pFlac)); + pFlac->bs = pInit->bs; + pFlac->onMeta = pInit->onMeta; + pFlac->pUserDataMD = pInit->pUserDataMD; + pFlac->maxBlockSizeInPCMFrames = pInit->maxBlockSizeInPCMFrames; + pFlac->sampleRate = pInit->sampleRate; + pFlac->channels = (drflac_uint8)pInit->channels; + pFlac->bitsPerSample = (drflac_uint8)pInit->bitsPerSample; + pFlac->totalPCMFrameCount = pInit->totalPCMFrameCount; + pFlac->container = pInit->container; +} + + +static drflac* drflac_open_with_metadata_private(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, void* pUserDataMD, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac_init_info init; + drflac_uint32 allocationSize; + drflac_uint32 wholeSIMDVectorCountPerChannel; + drflac_uint32 decodedSamplesAllocationSize; +#ifndef DR_FLAC_NO_OGG + drflac_oggbs* pOggbs = NULL; +#endif + drflac_uint64 firstFramePos; + drflac_uint64 seektablePos; + drflac_uint32 seekpointCount; + drflac_allocation_callbacks allocationCallbacks; + drflac* pFlac; + + /* CPU support first. */ + drflac__init_cpu_caps(); + + if (!drflac__init_private(&init, onRead, onSeek, onMeta, container, pUserData, pUserDataMD)) { + return NULL; + } + + if (pAllocationCallbacks != NULL) { + allocationCallbacks = *pAllocationCallbacks; + if (allocationCallbacks.onFree == NULL || (allocationCallbacks.onMalloc == NULL && allocationCallbacks.onRealloc == NULL)) { + return NULL; /* Invalid allocation callbacks. */ + } + } else { + allocationCallbacks.pUserData = NULL; + allocationCallbacks.onMalloc = drflac__malloc_default; + allocationCallbacks.onRealloc = drflac__realloc_default; + allocationCallbacks.onFree = drflac__free_default; + } + + + /* + The size of the allocation for the drflac object needs to be large enough to fit the following: + 1) The main members of the drflac structure + 2) A block of memory large enough to store the decoded samples of the largest frame in the stream + 3) If the container is Ogg, a drflac_oggbs object + + The complicated part of the allocation is making sure there's enough room the decoded samples, taking into consideration + the different SIMD instruction sets. + */ + allocationSize = sizeof(drflac); + + /* + The allocation size for decoded frames depends on the number of 32-bit integers that fit inside the largest SIMD vector + we are supporting. + */ + if ((init.maxBlockSizeInPCMFrames % (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))) == 0) { + wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))); + } else { + wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (DRFLAC_MAX_SIMD_VECTOR_SIZE / sizeof(drflac_int32))) + 1; + } + + decodedSamplesAllocationSize = wholeSIMDVectorCountPerChannel * DRFLAC_MAX_SIMD_VECTOR_SIZE * init.channels; + + allocationSize += decodedSamplesAllocationSize; + allocationSize += DRFLAC_MAX_SIMD_VECTOR_SIZE; /* Allocate extra bytes to ensure we have enough for alignment. */ + +#ifndef DR_FLAC_NO_OGG + /* There's additional data required for Ogg streams. */ + if (init.container == drflac_container_ogg) { + allocationSize += sizeof(drflac_oggbs); + + pOggbs = (drflac_oggbs*)drflac__malloc_from_callbacks(sizeof(*pOggbs), &allocationCallbacks); + if (pOggbs == NULL) { + return NULL; /*DRFLAC_OUT_OF_MEMORY;*/ + } + + DRFLAC_ZERO_MEMORY(pOggbs, sizeof(*pOggbs)); + pOggbs->onRead = onRead; + pOggbs->onSeek = onSeek; + pOggbs->pUserData = pUserData; + pOggbs->currentBytePos = init.oggFirstBytePos; + pOggbs->firstBytePos = init.oggFirstBytePos; + pOggbs->serialNumber = init.oggSerial; + pOggbs->bosPageHeader = init.oggBosHeader; + pOggbs->bytesRemainingInPage = 0; + } +#endif + + /* + This part is a bit awkward. We need to load the seektable so that it can be referenced in-memory, but I want the drflac object to + consist of only a single heap allocation. To this, the size of the seek table needs to be known, which we determine when reading + and decoding the metadata. + */ + firstFramePos = 42; /* <-- We know we are at byte 42 at this point. */ + seektablePos = 0; + seekpointCount = 0; + if (init.hasMetadataBlocks) { + drflac_read_proc onReadOverride = onRead; + drflac_seek_proc onSeekOverride = onSeek; + void* pUserDataOverride = pUserData; + +#ifndef DR_FLAC_NO_OGG + if (init.container == drflac_container_ogg) { + onReadOverride = drflac__on_read_ogg; + onSeekOverride = drflac__on_seek_ogg; + pUserDataOverride = (void*)pOggbs; + } +#endif + + if (!drflac__read_and_decode_metadata(onReadOverride, onSeekOverride, onMeta, pUserDataOverride, pUserDataMD, &firstFramePos, &seektablePos, &seekpointCount, &allocationCallbacks)) { + #ifndef DR_FLAC_NO_OGG + drflac__free_from_callbacks(pOggbs, &allocationCallbacks); + #endif + return NULL; + } + + allocationSize += seekpointCount * sizeof(drflac_seekpoint); + } + + + pFlac = (drflac*)drflac__malloc_from_callbacks(allocationSize, &allocationCallbacks); + if (pFlac == NULL) { + #ifndef DR_FLAC_NO_OGG + drflac__free_from_callbacks(pOggbs, &allocationCallbacks); + #endif + return NULL; + } + + drflac__init_from_info(pFlac, &init); + pFlac->allocationCallbacks = allocationCallbacks; + pFlac->pDecodedSamples = (drflac_int32*)drflac_align((size_t)pFlac->pExtraData, DRFLAC_MAX_SIMD_VECTOR_SIZE); + +#ifndef DR_FLAC_NO_OGG + if (init.container == drflac_container_ogg) { + drflac_oggbs* pInternalOggbs = (drflac_oggbs*)((drflac_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize + (seekpointCount * sizeof(drflac_seekpoint))); + DRFLAC_COPY_MEMORY(pInternalOggbs, pOggbs, sizeof(*pOggbs)); + + /* At this point the pOggbs object has been handed over to pInternalOggbs and can be freed. */ + drflac__free_from_callbacks(pOggbs, &allocationCallbacks); + pOggbs = NULL; + + /* The Ogg bistream needs to be layered on top of the original bitstream. */ + pFlac->bs.onRead = drflac__on_read_ogg; + pFlac->bs.onSeek = drflac__on_seek_ogg; + pFlac->bs.pUserData = (void*)pInternalOggbs; + pFlac->_oggbs = (void*)pInternalOggbs; + } +#endif + + pFlac->firstFLACFramePosInBytes = firstFramePos; + + /* NOTE: Seektables are not currently compatible with Ogg encapsulation (Ogg has its own accelerated seeking system). I may change this later, so I'm leaving this here for now. */ +#ifndef DR_FLAC_NO_OGG + if (init.container == drflac_container_ogg) + { + pFlac->pSeekpoints = NULL; + pFlac->seekpointCount = 0; + } + else +#endif + { + /* If we have a seektable we need to load it now, making sure we move back to where we were previously. */ + if (seektablePos != 0) { + pFlac->seekpointCount = seekpointCount; + pFlac->pSeekpoints = (drflac_seekpoint*)((drflac_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize); + + DRFLAC_ASSERT(pFlac->bs.onSeek != NULL); + DRFLAC_ASSERT(pFlac->bs.onRead != NULL); + + /* Seek to the seektable, then just read directly into our seektable buffer. */ + if (pFlac->bs.onSeek(pFlac->bs.pUserData, (int)seektablePos, drflac_seek_origin_start)) { + drflac_uint32 iSeekpoint; + + for (iSeekpoint = 0; iSeekpoint < seekpointCount; iSeekpoint += 1) { + if (pFlac->bs.onRead(pFlac->bs.pUserData, pFlac->pSeekpoints + iSeekpoint, DRFLAC_SEEKPOINT_SIZE_IN_BYTES) == DRFLAC_SEEKPOINT_SIZE_IN_BYTES) { + /* Endian swap. */ + pFlac->pSeekpoints[iSeekpoint].firstPCMFrame = drflac__be2host_64(pFlac->pSeekpoints[iSeekpoint].firstPCMFrame); + pFlac->pSeekpoints[iSeekpoint].flacFrameOffset = drflac__be2host_64(pFlac->pSeekpoints[iSeekpoint].flacFrameOffset); + pFlac->pSeekpoints[iSeekpoint].pcmFrameCount = drflac__be2host_16(pFlac->pSeekpoints[iSeekpoint].pcmFrameCount); + } else { + /* Failed to read the seektable. Pretend we don't have one. */ + pFlac->pSeekpoints = NULL; + pFlac->seekpointCount = 0; + break; + } + } + + /* We need to seek back to where we were. If this fails it's a critical error. */ + if (!pFlac->bs.onSeek(pFlac->bs.pUserData, (int)pFlac->firstFLACFramePosInBytes, drflac_seek_origin_start)) { + drflac__free_from_callbacks(pFlac, &allocationCallbacks); + return NULL; + } + } else { + /* Failed to seek to the seektable. Ominous sign, but for now we can just pretend we don't have one. */ + pFlac->pSeekpoints = NULL; + pFlac->seekpointCount = 0; + } + } + } + + + /* + If we get here, but don't have a STREAMINFO block, it means we've opened the stream in relaxed mode and need to decode + the first frame. + */ + if (!init.hasStreamInfoBlock) { + pFlac->currentFLACFrame.header = init.firstFrameHeader; + for (;;) { + drflac_result result = drflac__decode_flac_frame(pFlac); + if (result == DRFLAC_SUCCESS) { + break; + } else { + if (result == DRFLAC_CRC_MISMATCH) { + if (!drflac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + drflac__free_from_callbacks(pFlac, &allocationCallbacks); + return NULL; + } + continue; + } else { + drflac__free_from_callbacks(pFlac, &allocationCallbacks); + return NULL; + } + } + } + } + + return pFlac; +} + + + +#ifndef DR_FLAC_NO_STDIO +#include +#ifndef DR_FLAC_NO_WCHAR +#include /* For wcslen(), wcsrtombs() */ +#endif + +/* Errno */ +/* drflac_result_from_errno() is only used for fopen() and wfopen() so putting it inside DR_WAV_NO_STDIO for now. If something else needs this later we can move it out. */ +#include +static drflac_result drflac_result_from_errno(int e) +{ + switch (e) + { + case 0: return DRFLAC_SUCCESS; + #ifdef EPERM + case EPERM: return DRFLAC_INVALID_OPERATION; + #endif + #ifdef ENOENT + case ENOENT: return DRFLAC_DOES_NOT_EXIST; + #endif + #ifdef ESRCH + case ESRCH: return DRFLAC_DOES_NOT_EXIST; + #endif + #ifdef EINTR + case EINTR: return DRFLAC_INTERRUPT; + #endif + #ifdef EIO + case EIO: return DRFLAC_IO_ERROR; + #endif + #ifdef ENXIO + case ENXIO: return DRFLAC_DOES_NOT_EXIST; + #endif + #ifdef E2BIG + case E2BIG: return DRFLAC_INVALID_ARGS; + #endif + #ifdef ENOEXEC + case ENOEXEC: return DRFLAC_INVALID_FILE; + #endif + #ifdef EBADF + case EBADF: return DRFLAC_INVALID_FILE; + #endif + #ifdef ECHILD + case ECHILD: return DRFLAC_ERROR; + #endif + #ifdef EAGAIN + case EAGAIN: return DRFLAC_UNAVAILABLE; + #endif + #ifdef ENOMEM + case ENOMEM: return DRFLAC_OUT_OF_MEMORY; + #endif + #ifdef EACCES + case EACCES: return DRFLAC_ACCESS_DENIED; + #endif + #ifdef EFAULT + case EFAULT: return DRFLAC_BAD_ADDRESS; + #endif + #ifdef ENOTBLK + case ENOTBLK: return DRFLAC_ERROR; + #endif + #ifdef EBUSY + case EBUSY: return DRFLAC_BUSY; + #endif + #ifdef EEXIST + case EEXIST: return DRFLAC_ALREADY_EXISTS; + #endif + #ifdef EXDEV + case EXDEV: return DRFLAC_ERROR; + #endif + #ifdef ENODEV + case ENODEV: return DRFLAC_DOES_NOT_EXIST; + #endif + #ifdef ENOTDIR + case ENOTDIR: return DRFLAC_NOT_DIRECTORY; + #endif + #ifdef EISDIR + case EISDIR: return DRFLAC_IS_DIRECTORY; + #endif + #ifdef EINVAL + case EINVAL: return DRFLAC_INVALID_ARGS; + #endif + #ifdef ENFILE + case ENFILE: return DRFLAC_TOO_MANY_OPEN_FILES; + #endif + #ifdef EMFILE + case EMFILE: return DRFLAC_TOO_MANY_OPEN_FILES; + #endif + #ifdef ENOTTY + case ENOTTY: return DRFLAC_INVALID_OPERATION; + #endif + #ifdef ETXTBSY + case ETXTBSY: return DRFLAC_BUSY; + #endif + #ifdef EFBIG + case EFBIG: return DRFLAC_TOO_BIG; + #endif + #ifdef ENOSPC + case ENOSPC: return DRFLAC_NO_SPACE; + #endif + #ifdef ESPIPE + case ESPIPE: return DRFLAC_BAD_SEEK; + #endif + #ifdef EROFS + case EROFS: return DRFLAC_ACCESS_DENIED; + #endif + #ifdef EMLINK + case EMLINK: return DRFLAC_TOO_MANY_LINKS; + #endif + #ifdef EPIPE + case EPIPE: return DRFLAC_BAD_PIPE; + #endif + #ifdef EDOM + case EDOM: return DRFLAC_OUT_OF_RANGE; + #endif + #ifdef ERANGE + case ERANGE: return DRFLAC_OUT_OF_RANGE; + #endif + #ifdef EDEADLK + case EDEADLK: return DRFLAC_DEADLOCK; + #endif + #ifdef ENAMETOOLONG + case ENAMETOOLONG: return DRFLAC_PATH_TOO_LONG; + #endif + #ifdef ENOLCK + case ENOLCK: return DRFLAC_ERROR; + #endif + #ifdef ENOSYS + case ENOSYS: return DRFLAC_NOT_IMPLEMENTED; + #endif + #ifdef ENOTEMPTY + case ENOTEMPTY: return DRFLAC_DIRECTORY_NOT_EMPTY; + #endif + #ifdef ELOOP + case ELOOP: return DRFLAC_TOO_MANY_LINKS; + #endif + #ifdef ENOMSG + case ENOMSG: return DRFLAC_NO_MESSAGE; + #endif + #ifdef EIDRM + case EIDRM: return DRFLAC_ERROR; + #endif + #ifdef ECHRNG + case ECHRNG: return DRFLAC_ERROR; + #endif + #ifdef EL2NSYNC + case EL2NSYNC: return DRFLAC_ERROR; + #endif + #ifdef EL3HLT + case EL3HLT: return DRFLAC_ERROR; + #endif + #ifdef EL3RST + case EL3RST: return DRFLAC_ERROR; + #endif + #ifdef ELNRNG + case ELNRNG: return DRFLAC_OUT_OF_RANGE; + #endif + #ifdef EUNATCH + case EUNATCH: return DRFLAC_ERROR; + #endif + #ifdef ENOCSI + case ENOCSI: return DRFLAC_ERROR; + #endif + #ifdef EL2HLT + case EL2HLT: return DRFLAC_ERROR; + #endif + #ifdef EBADE + case EBADE: return DRFLAC_ERROR; + #endif + #ifdef EBADR + case EBADR: return DRFLAC_ERROR; + #endif + #ifdef EXFULL + case EXFULL: return DRFLAC_ERROR; + #endif + #ifdef ENOANO + case ENOANO: return DRFLAC_ERROR; + #endif + #ifdef EBADRQC + case EBADRQC: return DRFLAC_ERROR; + #endif + #ifdef EBADSLT + case EBADSLT: return DRFLAC_ERROR; + #endif + #ifdef EBFONT + case EBFONT: return DRFLAC_INVALID_FILE; + #endif + #ifdef ENOSTR + case ENOSTR: return DRFLAC_ERROR; + #endif + #ifdef ENODATA + case ENODATA: return DRFLAC_NO_DATA_AVAILABLE; + #endif + #ifdef ETIME + case ETIME: return DRFLAC_TIMEOUT; + #endif + #ifdef ENOSR + case ENOSR: return DRFLAC_NO_DATA_AVAILABLE; + #endif + #ifdef ENONET + case ENONET: return DRFLAC_NO_NETWORK; + #endif + #ifdef ENOPKG + case ENOPKG: return DRFLAC_ERROR; + #endif + #ifdef EREMOTE + case EREMOTE: return DRFLAC_ERROR; + #endif + #ifdef ENOLINK + case ENOLINK: return DRFLAC_ERROR; + #endif + #ifdef EADV + case EADV: return DRFLAC_ERROR; + #endif + #ifdef ESRMNT + case ESRMNT: return DRFLAC_ERROR; + #endif + #ifdef ECOMM + case ECOMM: return DRFLAC_ERROR; + #endif + #ifdef EPROTO + case EPROTO: return DRFLAC_ERROR; + #endif + #ifdef EMULTIHOP + case EMULTIHOP: return DRFLAC_ERROR; + #endif + #ifdef EDOTDOT + case EDOTDOT: return DRFLAC_ERROR; + #endif + #ifdef EBADMSG + case EBADMSG: return DRFLAC_BAD_MESSAGE; + #endif + #ifdef EOVERFLOW + case EOVERFLOW: return DRFLAC_TOO_BIG; + #endif + #ifdef ENOTUNIQ + case ENOTUNIQ: return DRFLAC_NOT_UNIQUE; + #endif + #ifdef EBADFD + case EBADFD: return DRFLAC_ERROR; + #endif + #ifdef EREMCHG + case EREMCHG: return DRFLAC_ERROR; + #endif + #ifdef ELIBACC + case ELIBACC: return DRFLAC_ACCESS_DENIED; + #endif + #ifdef ELIBBAD + case ELIBBAD: return DRFLAC_INVALID_FILE; + #endif + #ifdef ELIBSCN + case ELIBSCN: return DRFLAC_INVALID_FILE; + #endif + #ifdef ELIBMAX + case ELIBMAX: return DRFLAC_ERROR; + #endif + #ifdef ELIBEXEC + case ELIBEXEC: return DRFLAC_ERROR; + #endif + #ifdef EILSEQ + case EILSEQ: return DRFLAC_INVALID_DATA; + #endif + #ifdef ERESTART + case ERESTART: return DRFLAC_ERROR; + #endif + #ifdef ESTRPIPE + case ESTRPIPE: return DRFLAC_ERROR; + #endif + #ifdef EUSERS + case EUSERS: return DRFLAC_ERROR; + #endif + #ifdef ENOTSOCK + case ENOTSOCK: return DRFLAC_NOT_SOCKET; + #endif + #ifdef EDESTADDRREQ + case EDESTADDRREQ: return DRFLAC_NO_ADDRESS; + #endif + #ifdef EMSGSIZE + case EMSGSIZE: return DRFLAC_TOO_BIG; + #endif + #ifdef EPROTOTYPE + case EPROTOTYPE: return DRFLAC_BAD_PROTOCOL; + #endif + #ifdef ENOPROTOOPT + case ENOPROTOOPT: return DRFLAC_PROTOCOL_UNAVAILABLE; + #endif + #ifdef EPROTONOSUPPORT + case EPROTONOSUPPORT: return DRFLAC_PROTOCOL_NOT_SUPPORTED; + #endif + #ifdef ESOCKTNOSUPPORT + case ESOCKTNOSUPPORT: return DRFLAC_SOCKET_NOT_SUPPORTED; + #endif + #ifdef EOPNOTSUPP + case EOPNOTSUPP: return DRFLAC_INVALID_OPERATION; + #endif + #ifdef EPFNOSUPPORT + case EPFNOSUPPORT: return DRFLAC_PROTOCOL_FAMILY_NOT_SUPPORTED; + #endif + #ifdef EAFNOSUPPORT + case EAFNOSUPPORT: return DRFLAC_ADDRESS_FAMILY_NOT_SUPPORTED; + #endif + #ifdef EADDRINUSE + case EADDRINUSE: return DRFLAC_ALREADY_IN_USE; + #endif + #ifdef EADDRNOTAVAIL + case EADDRNOTAVAIL: return DRFLAC_ERROR; + #endif + #ifdef ENETDOWN + case ENETDOWN: return DRFLAC_NO_NETWORK; + #endif + #ifdef ENETUNREACH + case ENETUNREACH: return DRFLAC_NO_NETWORK; + #endif + #ifdef ENETRESET + case ENETRESET: return DRFLAC_NO_NETWORK; + #endif + #ifdef ECONNABORTED + case ECONNABORTED: return DRFLAC_NO_NETWORK; + #endif + #ifdef ECONNRESET + case ECONNRESET: return DRFLAC_CONNECTION_RESET; + #endif + #ifdef ENOBUFS + case ENOBUFS: return DRFLAC_NO_SPACE; + #endif + #ifdef EISCONN + case EISCONN: return DRFLAC_ALREADY_CONNECTED; + #endif + #ifdef ENOTCONN + case ENOTCONN: return DRFLAC_NOT_CONNECTED; + #endif + #ifdef ESHUTDOWN + case ESHUTDOWN: return DRFLAC_ERROR; + #endif + #ifdef ETOOMANYREFS + case ETOOMANYREFS: return DRFLAC_ERROR; + #endif + #ifdef ETIMEDOUT + case ETIMEDOUT: return DRFLAC_TIMEOUT; + #endif + #ifdef ECONNREFUSED + case ECONNREFUSED: return DRFLAC_CONNECTION_REFUSED; + #endif + #ifdef EHOSTDOWN + case EHOSTDOWN: return DRFLAC_NO_HOST; + #endif + #ifdef EHOSTUNREACH + case EHOSTUNREACH: return DRFLAC_NO_HOST; + #endif + #ifdef EALREADY + case EALREADY: return DRFLAC_IN_PROGRESS; + #endif + #ifdef EINPROGRESS + case EINPROGRESS: return DRFLAC_IN_PROGRESS; + #endif + #ifdef ESTALE + case ESTALE: return DRFLAC_INVALID_FILE; + #endif + #ifdef EUCLEAN + case EUCLEAN: return DRFLAC_ERROR; + #endif + #ifdef ENOTNAM + case ENOTNAM: return DRFLAC_ERROR; + #endif + #ifdef ENAVAIL + case ENAVAIL: return DRFLAC_ERROR; + #endif + #ifdef EISNAM + case EISNAM: return DRFLAC_ERROR; + #endif + #ifdef EREMOTEIO + case EREMOTEIO: return DRFLAC_IO_ERROR; + #endif + #ifdef EDQUOT + case EDQUOT: return DRFLAC_NO_SPACE; + #endif + #ifdef ENOMEDIUM + case ENOMEDIUM: return DRFLAC_DOES_NOT_EXIST; + #endif + #ifdef EMEDIUMTYPE + case EMEDIUMTYPE: return DRFLAC_ERROR; + #endif + #ifdef ECANCELED + case ECANCELED: return DRFLAC_CANCELLED; + #endif + #ifdef ENOKEY + case ENOKEY: return DRFLAC_ERROR; + #endif + #ifdef EKEYEXPIRED + case EKEYEXPIRED: return DRFLAC_ERROR; + #endif + #ifdef EKEYREVOKED + case EKEYREVOKED: return DRFLAC_ERROR; + #endif + #ifdef EKEYREJECTED + case EKEYREJECTED: return DRFLAC_ERROR; + #endif + #ifdef EOWNERDEAD + case EOWNERDEAD: return DRFLAC_ERROR; + #endif + #ifdef ENOTRECOVERABLE + case ENOTRECOVERABLE: return DRFLAC_ERROR; + #endif + #ifdef ERFKILL + case ERFKILL: return DRFLAC_ERROR; + #endif + #ifdef EHWPOISON + case EHWPOISON: return DRFLAC_ERROR; + #endif + default: return DRFLAC_ERROR; + } +} +/* End Errno */ + +/* fopen */ +static drflac_result drflac_fopen(FILE** ppFile, const char* pFilePath, const char* pOpenMode) +{ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + errno_t err; +#endif + + if (ppFile != NULL) { + *ppFile = NULL; /* Safety. */ + } + + if (pFilePath == NULL || pOpenMode == NULL || ppFile == NULL) { + return DRFLAC_INVALID_ARGS; + } + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + err = fopen_s(ppFile, pFilePath, pOpenMode); + if (err != 0) { + return drflac_result_from_errno(err); + } +#else +#if defined(_WIN32) || defined(__APPLE__) + *ppFile = fopen(pFilePath, pOpenMode); +#else + #if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 64 && defined(_LARGEFILE64_SOURCE) + *ppFile = fopen64(pFilePath, pOpenMode); + #else + *ppFile = fopen(pFilePath, pOpenMode); + #endif +#endif + if (*ppFile == NULL) { + drflac_result result = drflac_result_from_errno(errno); + if (result == DRFLAC_SUCCESS) { + result = DRFLAC_ERROR; /* Just a safety check to make sure we never ever return success when pFile == NULL. */ + } + + return result; + } +#endif + + return DRFLAC_SUCCESS; +} + +/* +_wfopen() isn't always available in all compilation environments. + + * Windows only. + * MSVC seems to support it universally as far back as VC6 from what I can tell (haven't checked further back). + * MinGW-64 (both 32- and 64-bit) seems to support it. + * MinGW wraps it in !defined(__STRICT_ANSI__). + * OpenWatcom wraps it in !defined(_NO_EXT_KEYS). + +This can be reviewed as compatibility issues arise. The preference is to use _wfopen_s() and _wfopen() as opposed to the wcsrtombs() +fallback, so if you notice your compiler not detecting this properly I'm happy to look at adding support. +*/ +#if defined(_WIN32) + #if defined(_MSC_VER) || defined(__MINGW64__) || (!defined(__STRICT_ANSI__) && !defined(_NO_EXT_KEYS)) + #define DRFLAC_HAS_WFOPEN + #endif +#endif + +#ifndef DR_FLAC_NO_WCHAR +static drflac_result drflac_wfopen(FILE** ppFile, const wchar_t* pFilePath, const wchar_t* pOpenMode, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + if (ppFile != NULL) { + *ppFile = NULL; /* Safety. */ + } + + if (pFilePath == NULL || pOpenMode == NULL || ppFile == NULL) { + return DRFLAC_INVALID_ARGS; + } + +#if defined(DRFLAC_HAS_WFOPEN) + { + /* Use _wfopen() on Windows. */ + #if defined(_MSC_VER) && _MSC_VER >= 1400 + errno_t err = _wfopen_s(ppFile, pFilePath, pOpenMode); + if (err != 0) { + return drflac_result_from_errno(err); + } + #else + *ppFile = _wfopen(pFilePath, pOpenMode); + if (*ppFile == NULL) { + return drflac_result_from_errno(errno); + } + #endif + (void)pAllocationCallbacks; + } +#else + /* + Use fopen() on anything other than Windows. Requires a conversion. This is annoying because + fopen() is locale specific. The only real way I can think of to do this is with wcsrtombs(). Note + that wcstombs() is apparently not thread-safe because it uses a static global mbstate_t object for + maintaining state. I've checked this with -std=c89 and it works, but if somebody get's a compiler + error I'll look into improving compatibility. + */ + + /* + Some compilers don't support wchar_t or wcsrtombs() which we're using below. In this case we just + need to abort with an error. If you encounter a compiler lacking such support, add it to this list + and submit a bug report and it'll be added to the library upstream. + */ + #if defined(__DJGPP__) + { + /* Nothing to do here. This will fall through to the error check below. */ + } + #else + { + mbstate_t mbs; + size_t lenMB; + const wchar_t* pFilePathTemp = pFilePath; + char* pFilePathMB = NULL; + char pOpenModeMB[32] = {0}; + + /* Get the length first. */ + DRFLAC_ZERO_OBJECT(&mbs); + lenMB = wcsrtombs(NULL, &pFilePathTemp, 0, &mbs); + if (lenMB == (size_t)-1) { + return drflac_result_from_errno(errno); + } + + pFilePathMB = (char*)drflac__malloc_from_callbacks(lenMB + 1, pAllocationCallbacks); + if (pFilePathMB == NULL) { + return DRFLAC_OUT_OF_MEMORY; + } + + pFilePathTemp = pFilePath; + DRFLAC_ZERO_OBJECT(&mbs); + wcsrtombs(pFilePathMB, &pFilePathTemp, lenMB + 1, &mbs); + + /* The open mode should always consist of ASCII characters so we should be able to do a trivial conversion. */ + { + size_t i = 0; + for (;;) { + if (pOpenMode[i] == 0) { + pOpenModeMB[i] = '\0'; + break; + } + + pOpenModeMB[i] = (char)pOpenMode[i]; + i += 1; + } + } + + *ppFile = fopen(pFilePathMB, pOpenModeMB); + + drflac__free_from_callbacks(pFilePathMB, pAllocationCallbacks); + } + #endif + + if (*ppFile == NULL) { + return DRFLAC_ERROR; + } +#endif + + return DRFLAC_SUCCESS; +} +#endif +/* End fopen */ + +static size_t drflac__on_read_stdio(void* pUserData, void* bufferOut, size_t bytesToRead) +{ + return fread(bufferOut, 1, bytesToRead, (FILE*)pUserData); +} + +static drflac_bool32 drflac__on_seek_stdio(void* pUserData, int offset, drflac_seek_origin origin) +{ + DRFLAC_ASSERT(offset >= 0); /* <-- Never seek backwards. */ + + return fseek((FILE*)pUserData, offset, (origin == drflac_seek_origin_current) ? SEEK_CUR : SEEK_SET) == 0; +} + + +DRFLAC_API drflac* drflac_open_file(const char* pFileName, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + FILE* pFile; + + if (drflac_fopen(&pFile, pFileName, "rb") != DRFLAC_SUCCESS) { + return NULL; + } + + pFlac = drflac_open(drflac__on_read_stdio, drflac__on_seek_stdio, (void*)pFile, pAllocationCallbacks); + if (pFlac == NULL) { + fclose(pFile); + return NULL; + } + + return pFlac; +} + +#ifndef DR_FLAC_NO_WCHAR +DRFLAC_API drflac* drflac_open_file_w(const wchar_t* pFileName, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + FILE* pFile; + + if (drflac_wfopen(&pFile, pFileName, L"rb", pAllocationCallbacks) != DRFLAC_SUCCESS) { + return NULL; + } + + pFlac = drflac_open(drflac__on_read_stdio, drflac__on_seek_stdio, (void*)pFile, pAllocationCallbacks); + if (pFlac == NULL) { + fclose(pFile); + return NULL; + } + + return pFlac; +} +#endif + +DRFLAC_API drflac* drflac_open_file_with_metadata(const char* pFileName, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + FILE* pFile; + + if (drflac_fopen(&pFile, pFileName, "rb") != DRFLAC_SUCCESS) { + return NULL; + } + + pFlac = drflac_open_with_metadata_private(drflac__on_read_stdio, drflac__on_seek_stdio, onMeta, drflac_container_unknown, (void*)pFile, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + fclose(pFile); + return pFlac; + } + + return pFlac; +} + +#ifndef DR_FLAC_NO_WCHAR +DRFLAC_API drflac* drflac_open_file_with_metadata_w(const wchar_t* pFileName, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + FILE* pFile; + + if (drflac_wfopen(&pFile, pFileName, L"rb", pAllocationCallbacks) != DRFLAC_SUCCESS) { + return NULL; + } + + pFlac = drflac_open_with_metadata_private(drflac__on_read_stdio, drflac__on_seek_stdio, onMeta, drflac_container_unknown, (void*)pFile, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + fclose(pFile); + return pFlac; + } + + return pFlac; +} +#endif +#endif /* DR_FLAC_NO_STDIO */ + +static size_t drflac__on_read_memory(void* pUserData, void* bufferOut, size_t bytesToRead) +{ + drflac__memory_stream* memoryStream = (drflac__memory_stream*)pUserData; + size_t bytesRemaining; + + DRFLAC_ASSERT(memoryStream != NULL); + DRFLAC_ASSERT(memoryStream->dataSize >= memoryStream->currentReadPos); + + bytesRemaining = memoryStream->dataSize - memoryStream->currentReadPos; + if (bytesToRead > bytesRemaining) { + bytesToRead = bytesRemaining; + } + + if (bytesToRead > 0) { + DRFLAC_COPY_MEMORY(bufferOut, memoryStream->data + memoryStream->currentReadPos, bytesToRead); + memoryStream->currentReadPos += bytesToRead; + } + + return bytesToRead; +} + +static drflac_bool32 drflac__on_seek_memory(void* pUserData, int offset, drflac_seek_origin origin) +{ + drflac__memory_stream* memoryStream = (drflac__memory_stream*)pUserData; + + DRFLAC_ASSERT(memoryStream != NULL); + DRFLAC_ASSERT(offset >= 0); /* <-- Never seek backwards. */ + + if (offset > (drflac_int64)memoryStream->dataSize) { + return DRFLAC_FALSE; + } + + if (origin == drflac_seek_origin_current) { + if (memoryStream->currentReadPos + offset <= memoryStream->dataSize) { + memoryStream->currentReadPos += offset; + } else { + return DRFLAC_FALSE; /* Trying to seek too far forward. */ + } + } else { + if ((drflac_uint32)offset <= memoryStream->dataSize) { + memoryStream->currentReadPos = offset; + } else { + return DRFLAC_FALSE; /* Trying to seek too far forward. */ + } + } + + return DRFLAC_TRUE; +} + +DRFLAC_API drflac* drflac_open_memory(const void* pData, size_t dataSize, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac__memory_stream memoryStream; + drflac* pFlac; + + memoryStream.data = (const drflac_uint8*)pData; + memoryStream.dataSize = dataSize; + memoryStream.currentReadPos = 0; + pFlac = drflac_open(drflac__on_read_memory, drflac__on_seek_memory, &memoryStream, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + pFlac->memoryStream = memoryStream; + + /* This is an awful hack... */ +#ifndef DR_FLAC_NO_OGG + if (pFlac->container == drflac_container_ogg) + { + drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; + oggbs->pUserData = &pFlac->memoryStream; + } + else +#endif + { + pFlac->bs.pUserData = &pFlac->memoryStream; + } + + return pFlac; +} + +DRFLAC_API drflac* drflac_open_memory_with_metadata(const void* pData, size_t dataSize, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac__memory_stream memoryStream; + drflac* pFlac; + + memoryStream.data = (const drflac_uint8*)pData; + memoryStream.dataSize = dataSize; + memoryStream.currentReadPos = 0; + pFlac = drflac_open_with_metadata_private(drflac__on_read_memory, drflac__on_seek_memory, onMeta, drflac_container_unknown, &memoryStream, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + pFlac->memoryStream = memoryStream; + + /* This is an awful hack... */ +#ifndef DR_FLAC_NO_OGG + if (pFlac->container == drflac_container_ogg) + { + drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; + oggbs->pUserData = &pFlac->memoryStream; + } + else +#endif + { + pFlac->bs.pUserData = &pFlac->memoryStream; + } + + return pFlac; +} + + + +DRFLAC_API drflac* drflac_open(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + return drflac_open_with_metadata_private(onRead, onSeek, NULL, drflac_container_unknown, pUserData, pUserData, pAllocationCallbacks); +} +DRFLAC_API drflac* drflac_open_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_container container, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + return drflac_open_with_metadata_private(onRead, onSeek, NULL, container, pUserData, pUserData, pAllocationCallbacks); +} + +DRFLAC_API drflac* drflac_open_with_metadata(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + return drflac_open_with_metadata_private(onRead, onSeek, onMeta, drflac_container_unknown, pUserData, pUserData, pAllocationCallbacks); +} +DRFLAC_API drflac* drflac_open_with_metadata_relaxed(drflac_read_proc onRead, drflac_seek_proc onSeek, drflac_meta_proc onMeta, drflac_container container, void* pUserData, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + return drflac_open_with_metadata_private(onRead, onSeek, onMeta, container, pUserData, pUserData, pAllocationCallbacks); +} + +DRFLAC_API void drflac_close(drflac* pFlac) +{ + if (pFlac == NULL) { + return; + } + +#ifndef DR_FLAC_NO_STDIO + /* + If we opened the file with drflac_open_file() we will want to close the file handle. We can know whether or not drflac_open_file() + was used by looking at the callbacks. + */ + if (pFlac->bs.onRead == drflac__on_read_stdio) { + fclose((FILE*)pFlac->bs.pUserData); + } + +#ifndef DR_FLAC_NO_OGG + /* Need to clean up Ogg streams a bit differently due to the way the bit streaming is chained. */ + if (pFlac->container == drflac_container_ogg) { + drflac_oggbs* oggbs = (drflac_oggbs*)pFlac->_oggbs; + DRFLAC_ASSERT(pFlac->bs.onRead == drflac__on_read_ogg); + + if (oggbs->onRead == drflac__on_read_stdio) { + fclose((FILE*)oggbs->pUserData); + } + } +#endif +#endif + + drflac__free_from_callbacks(pFlac, &pFlac->allocationCallbacks); +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_left_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 left = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 side = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_left_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 left0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 left1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 left2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 left3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 right0 = left0 - side0; + drflac_uint32 right1 = left1 - side1; + drflac_uint32 right2 = left2 - side2; + drflac_uint32 right3 = left3 - side3; + + pOutputSamples[i*8+0] = (drflac_int32)left0; + pOutputSamples[i*8+1] = (drflac_int32)right0; + pOutputSamples[i*8+2] = (drflac_int32)left1; + pOutputSamples[i*8+3] = (drflac_int32)right1; + pOutputSamples[i*8+4] = (drflac_int32)left2; + pOutputSamples[i*8+5] = (drflac_int32)right2; + pOutputSamples[i*8+6] = (drflac_int32)left3; + pOutputSamples[i*8+7] = (drflac_int32)right3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_left_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + for (i = 0; i < frameCount4; ++i) { + __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i right = _mm_sub_epi32(left, side); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right)); + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_left_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t left; + uint32x4_t side; + uint32x4_t right; + + left = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + right = vsubq_u32(left, side); + + drflac__vst2q_u32((drflac_uint32*)pOutputSamples + i*8, vzipq_u32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_left_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s32__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s32__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_right_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 side = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 right = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_right_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 side0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 side1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 side2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 side3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 right0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 right1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 right2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 right3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 left0 = right0 + side0; + drflac_uint32 left1 = right1 + side1; + drflac_uint32 left2 = right2 + side2; + drflac_uint32 left3 = right3 + side3; + + pOutputSamples[i*8+0] = (drflac_int32)left0; + pOutputSamples[i*8+1] = (drflac_int32)right0; + pOutputSamples[i*8+2] = (drflac_int32)left1; + pOutputSamples[i*8+3] = (drflac_int32)right1; + pOutputSamples[i*8+4] = (drflac_int32)left2; + pOutputSamples[i*8+5] = (drflac_int32)right2; + pOutputSamples[i*8+6] = (drflac_int32)left3; + pOutputSamples[i*8+7] = (drflac_int32)right3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_right_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + for (i = 0; i < frameCount4; ++i) { + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i left = _mm_add_epi32(right, side); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right)); + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_right_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t side; + uint32x4_t right; + uint32x4_t left; + + side = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + right = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + left = vaddq_u32(right, side); + + drflac__vst2q_u32((drflac_uint32*)pOutputSamples + i*8, vzipq_u32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left; + pOutputSamples[i*2+1] = (drflac_int32)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_right_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s32__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s32__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_mid_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid + side) >> 1) << unusedBitsPerSample); + pOutputSamples[i*2+1] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid - side) >> 1) << unusedBitsPerSample); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_mid_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_int32 shift = unusedBitsPerSample; + + if (shift > 0) { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = (mid0 + side0) << shift; + temp1L = (mid1 + side1) << shift; + temp2L = (mid2 + side2) << shift; + temp3L = (mid3 + side3) << shift; + + temp0R = (mid0 - side0) << shift; + temp1R = (mid1 - side1) << shift; + temp2R = (mid2 - side2) << shift; + temp3R = (mid3 - side3) << shift; + + pOutputSamples[i*8+0] = (drflac_int32)temp0L; + pOutputSamples[i*8+1] = (drflac_int32)temp0R; + pOutputSamples[i*8+2] = (drflac_int32)temp1L; + pOutputSamples[i*8+3] = (drflac_int32)temp1R; + pOutputSamples[i*8+4] = (drflac_int32)temp2L; + pOutputSamples[i*8+5] = (drflac_int32)temp2R; + pOutputSamples[i*8+6] = (drflac_int32)temp3L; + pOutputSamples[i*8+7] = (drflac_int32)temp3R; + } + } else { + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = (drflac_uint32)((drflac_int32)(mid0 + side0) >> 1); + temp1L = (drflac_uint32)((drflac_int32)(mid1 + side1) >> 1); + temp2L = (drflac_uint32)((drflac_int32)(mid2 + side2) >> 1); + temp3L = (drflac_uint32)((drflac_int32)(mid3 + side3) >> 1); + + temp0R = (drflac_uint32)((drflac_int32)(mid0 - side0) >> 1); + temp1R = (drflac_uint32)((drflac_int32)(mid1 - side1) >> 1); + temp2R = (drflac_uint32)((drflac_int32)(mid2 - side2) >> 1); + temp3R = (drflac_uint32)((drflac_int32)(mid3 - side3) >> 1); + + pOutputSamples[i*8+0] = (drflac_int32)temp0L; + pOutputSamples[i*8+1] = (drflac_int32)temp0R; + pOutputSamples[i*8+2] = (drflac_int32)temp1L; + pOutputSamples[i*8+3] = (drflac_int32)temp1R; + pOutputSamples[i*8+4] = (drflac_int32)temp2L; + pOutputSamples[i*8+5] = (drflac_int32)temp2R; + pOutputSamples[i*8+6] = (drflac_int32)temp3L; + pOutputSamples[i*8+7] = (drflac_int32)temp3R; + } + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid + side) >> 1) << unusedBitsPerSample); + pOutputSamples[i*2+1] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid - side) >> 1) << unusedBitsPerSample); + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_mid_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_int32 shift = unusedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i left; + __m128i right; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + left = _mm_srai_epi32(_mm_add_epi32(mid, side), 1); + right = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right)); + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)(mid + side) >> 1; + pOutputSamples[i*2+1] = (drflac_int32)(mid - side) >> 1; + } + } else { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i left; + __m128i right; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + left = _mm_slli_epi32(_mm_add_epi32(mid, side), shift); + right = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right)); + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((mid + side) << shift); + pOutputSamples[i*2+1] = (drflac_int32)((mid - side) << shift); + } + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_mid_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_int32 shift = unusedBitsPerSample; + int32x4_t wbpsShift0_4; /* wbps = Wasted Bits Per Sample */ + int32x4_t wbpsShift1_4; /* wbps = Wasted Bits Per Sample */ + uint32x4_t one4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + one4 = vdupq_n_u32(1); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + uint32x4_t mid; + uint32x4_t side; + int32x4_t left; + int32x4_t right; + + mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, one4)); + + left = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1); + right = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1); + + drflac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)(mid + side) >> 1; + pOutputSamples[i*2+1] = (drflac_int32)(mid - side) >> 1; + } + } else { + int32x4_t shift4; + + shift -= 1; + shift4 = vdupq_n_s32(shift); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t mid; + uint32x4_t side; + int32x4_t left; + int32x4_t right; + + mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, one4)); + + left = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4)); + right = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4)); + + drflac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((mid + side) << shift); + pOutputSamples[i*2+1] = (drflac_int32)((mid - side) << shift); + } + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_mid_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s32__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s32__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)((drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample)); + pOutputSamples[i*2+1] = (drflac_int32)((drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample)); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1; + + pOutputSamples[i*8+0] = (drflac_int32)tempL0; + pOutputSamples[i*8+1] = (drflac_int32)tempR0; + pOutputSamples[i*8+2] = (drflac_int32)tempL1; + pOutputSamples[i*8+3] = (drflac_int32)tempR1; + pOutputSamples[i*8+4] = (drflac_int32)tempL2; + pOutputSamples[i*8+5] = (drflac_int32)tempR2; + pOutputSamples[i*8+6] = (drflac_int32)tempL3; + pOutputSamples[i*8+7] = (drflac_int32)tempR3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0); + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1); + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right)); + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0); + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1); + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + int32x4_t shift4_0 = vdupq_n_s32(shift0); + int32x4_t shift4_1 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + int32x4_t left; + int32x4_t right; + + left = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift4_0)); + right = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift4_1)); + + drflac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0); + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s32__decode_independent_stereo(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int32* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s32__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s32__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_s32(drflac* pFlac, drflac_uint64 framesToRead, drflac_int32* pBufferOut) +{ + drflac_uint64 framesRead; + drflac_uint32 unusedBitsPerSample; + + if (pFlac == NULL || framesToRead == 0) { + return 0; + } + + if (pBufferOut == NULL) { + return drflac__seek_forward_by_pcm_frames(pFlac, framesToRead); + } + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 32); + unusedBitsPerSample = 32 - pFlac->bitsPerSample; + + framesRead = 0; + while (framesToRead > 0) { + /* If we've run out of samples in this frame, go to the next. */ + if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_and_decode_next_flac_frame(pFlac)) { + break; /* Couldn't read the next frame, so just break from the loop and return. */ + } + } else { + unsigned int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + drflac_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining; + drflac_uint64 frameCountThisIteration = framesToRead; + + if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { + frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; + } + + if (channelCount == 2) { + const drflac_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; + const drflac_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; + + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + drflac_read_pcm_frames_s32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + drflac_read_pcm_frames_s32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + drflac_read_pcm_frames_s32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: + default: + { + drflac_read_pcm_frames_s32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + } + } else { + /* Generic interleaving. */ + drflac_uint64 i; + for (i = 0; i < frameCountThisIteration; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + pBufferOut[(i*channelCount)+j] = (drflac_int32)((drflac_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + } + } + } + + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; + pFlac->currentFLACFrame.pcmFramesRemaining -= (drflac_uint32)frameCountThisIteration; + } + } + + return framesRead; +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_left_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 left = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 side = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 right = left - side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_left_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 left0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 left1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 left2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 left3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 right0 = left0 - side0; + drflac_uint32 right1 = left1 - side1; + drflac_uint32 right2 = left2 - side2; + drflac_uint32 right3 = left3 - side3; + + left0 >>= 16; + left1 >>= 16; + left2 >>= 16; + left3 >>= 16; + + right0 >>= 16; + right1 >>= 16; + right2 >>= 16; + right3 >>= 16; + + pOutputSamples[i*8+0] = (drflac_int16)left0; + pOutputSamples[i*8+1] = (drflac_int16)right0; + pOutputSamples[i*8+2] = (drflac_int16)left1; + pOutputSamples[i*8+3] = (drflac_int16)right1; + pOutputSamples[i*8+4] = (drflac_int16)left2; + pOutputSamples[i*8+5] = (drflac_int16)right2; + pOutputSamples[i*8+6] = (drflac_int16)left3; + pOutputSamples[i*8+7] = (drflac_int16)right3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_left_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + for (i = 0; i < frameCount4; ++i) { + __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i right = _mm_sub_epi32(left, side); + + left = _mm_srai_epi32(left, 16); + right = _mm_srai_epi32(right, 16); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), drflac__mm_packs_interleaved_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_left_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t left; + uint32x4_t side; + uint32x4_t right; + + left = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + right = vsubq_u32(left, side); + + left = vshrq_n_u32(left, 16); + right = vshrq_n_u32(right, 16); + + drflac__vst2q_u16((drflac_uint16*)pOutputSamples + i*8, vzip_u16(vmovn_u32(left), vmovn_u32(right))); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_left_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s16__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s16__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_right_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 side = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 right = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 left = right + side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_right_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 side0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 side1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 side2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 side3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 right0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 right1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 right2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 right3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 left0 = right0 + side0; + drflac_uint32 left1 = right1 + side1; + drflac_uint32 left2 = right2 + side2; + drflac_uint32 left3 = right3 + side3; + + left0 >>= 16; + left1 >>= 16; + left2 >>= 16; + left3 >>= 16; + + right0 >>= 16; + right1 >>= 16; + right2 >>= 16; + right3 >>= 16; + + pOutputSamples[i*8+0] = (drflac_int16)left0; + pOutputSamples[i*8+1] = (drflac_int16)right0; + pOutputSamples[i*8+2] = (drflac_int16)left1; + pOutputSamples[i*8+3] = (drflac_int16)right1; + pOutputSamples[i*8+4] = (drflac_int16)left2; + pOutputSamples[i*8+5] = (drflac_int16)right2; + pOutputSamples[i*8+6] = (drflac_int16)left3; + pOutputSamples[i*8+7] = (drflac_int16)right3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_right_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + for (i = 0; i < frameCount4; ++i) { + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i left = _mm_add_epi32(right, side); + + left = _mm_srai_epi32(left, 16); + right = _mm_srai_epi32(right, 16); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), drflac__mm_packs_interleaved_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_right_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t side; + uint32x4_t right; + uint32x4_t left; + + side = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + right = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + left = vaddq_u32(right, side); + + left = vshrq_n_u32(left, 16); + right = vshrq_n_u32(right, 16); + + drflac__vst2q_u16((drflac_uint16*)pOutputSamples + i*8, vzip_u16(vmovn_u32(left), vmovn_u32(right))); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + left >>= 16; + right >>= 16; + + pOutputSamples[i*2+0] = (drflac_int16)left; + pOutputSamples[i*2+1] = (drflac_int16)right; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_right_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s16__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s16__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_mid_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + drflac_uint32 mid = (drflac_uint32)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = (drflac_uint32)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((drflac_uint32)((drflac_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((drflac_uint32)((drflac_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_mid_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample; + + if (shift > 0) { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = (mid0 + side0) << shift; + temp1L = (mid1 + side1) << shift; + temp2L = (mid2 + side2) << shift; + temp3L = (mid3 + side3) << shift; + + temp0R = (mid0 - side0) << shift; + temp1R = (mid1 - side1) << shift; + temp2R = (mid2 - side2) << shift; + temp3R = (mid3 - side3) << shift; + + temp0L >>= 16; + temp1L >>= 16; + temp2L >>= 16; + temp3L >>= 16; + + temp0R >>= 16; + temp1R >>= 16; + temp2R >>= 16; + temp3R >>= 16; + + pOutputSamples[i*8+0] = (drflac_int16)temp0L; + pOutputSamples[i*8+1] = (drflac_int16)temp0R; + pOutputSamples[i*8+2] = (drflac_int16)temp1L; + pOutputSamples[i*8+3] = (drflac_int16)temp1R; + pOutputSamples[i*8+4] = (drflac_int16)temp2L; + pOutputSamples[i*8+5] = (drflac_int16)temp2R; + pOutputSamples[i*8+6] = (drflac_int16)temp3L; + pOutputSamples[i*8+7] = (drflac_int16)temp3R; + } + } else { + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = ((drflac_int32)(mid0 + side0) >> 1); + temp1L = ((drflac_int32)(mid1 + side1) >> 1); + temp2L = ((drflac_int32)(mid2 + side2) >> 1); + temp3L = ((drflac_int32)(mid3 + side3) >> 1); + + temp0R = ((drflac_int32)(mid0 - side0) >> 1); + temp1R = ((drflac_int32)(mid1 - side1) >> 1); + temp2R = ((drflac_int32)(mid2 - side2) >> 1); + temp3R = ((drflac_int32)(mid3 - side3) >> 1); + + temp0L >>= 16; + temp1L >>= 16; + temp2L >>= 16; + temp3L >>= 16; + + temp0R >>= 16; + temp1R >>= 16; + temp2R >>= 16; + temp3R >>= 16; + + pOutputSamples[i*8+0] = (drflac_int16)temp0L; + pOutputSamples[i*8+1] = (drflac_int16)temp0R; + pOutputSamples[i*8+2] = (drflac_int16)temp1L; + pOutputSamples[i*8+3] = (drflac_int16)temp1R; + pOutputSamples[i*8+4] = (drflac_int16)temp2L; + pOutputSamples[i*8+5] = (drflac_int16)temp2R; + pOutputSamples[i*8+6] = (drflac_int16)temp3L; + pOutputSamples[i*8+7] = (drflac_int16)temp3R; + } + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((drflac_uint32)((drflac_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((drflac_uint32)((drflac_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16); + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_mid_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i left; + __m128i right; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + left = _mm_srai_epi32(_mm_add_epi32(mid, side), 1); + right = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1); + + left = _mm_srai_epi32(left, 16); + right = _mm_srai_epi32(right, 16); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), drflac__mm_packs_interleaved_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((drflac_int32)(mid + side) >> 1) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((drflac_int32)(mid - side) >> 1) >> 16); + } + } else { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i left; + __m128i right; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + left = _mm_slli_epi32(_mm_add_epi32(mid, side), shift); + right = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift); + + left = _mm_srai_epi32(left, 16); + right = _mm_srai_epi32(right, 16); + + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), drflac__mm_packs_interleaved_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((mid + side) << shift) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((mid - side) << shift) >> 16); + } + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_mid_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample; + int32x4_t wbpsShift0_4; /* wbps = Wasted Bits Per Sample */ + int32x4_t wbpsShift1_4; /* wbps = Wasted Bits Per Sample */ + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + uint32x4_t mid; + uint32x4_t side; + int32x4_t left; + int32x4_t right; + + mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1))); + + left = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1); + right = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1); + + left = vshrq_n_s32(left, 16); + right = vshrq_n_s32(right, 16); + + drflac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right))); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((drflac_int32)(mid + side) >> 1) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((drflac_int32)(mid - side) >> 1) >> 16); + } + } else { + int32x4_t shift4; + + shift -= 1; + shift4 = vdupq_n_s32(shift); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t mid; + uint32x4_t side; + int32x4_t left; + int32x4_t right; + + mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1))); + + left = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4)); + right = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4)); + + left = vshrq_n_s32(left, 16); + right = vshrq_n_s32(right, 16); + + drflac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right))); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int16)(((mid + side) << shift) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)(((mid - side) << shift) >> 16); + } + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_mid_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s16__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s16__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_independent_stereo__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int16)((drflac_int32)((drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample)) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)((drflac_int32)((drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample)) >> 16); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_independent_stereo__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1; + + tempL0 >>= 16; + tempL1 >>= 16; + tempL2 >>= 16; + tempL3 >>= 16; + + tempR0 >>= 16; + tempR1 >>= 16; + tempR2 >>= 16; + tempR3 >>= 16; + + pOutputSamples[i*8+0] = (drflac_int16)tempL0; + pOutputSamples[i*8+1] = (drflac_int16)tempR0; + pOutputSamples[i*8+2] = (drflac_int16)tempL1; + pOutputSamples[i*8+3] = (drflac_int16)tempR1; + pOutputSamples[i*8+4] = (drflac_int16)tempL2; + pOutputSamples[i*8+5] = (drflac_int16)tempR2; + pOutputSamples[i*8+6] = (drflac_int16)tempL3; + pOutputSamples[i*8+7] = (drflac_int16)tempR3; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int16)((pInputSamples0U32[i] << shift0) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)((pInputSamples1U32[i] << shift1) >> 16); + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_independent_stereo__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + for (i = 0; i < frameCount4; ++i) { + __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + + left = _mm_srai_epi32(left, 16); + right = _mm_srai_epi32(right, 16); + + /* At this point we have results. We can now pack and interleave these into a single __m128i object and then store the in the output buffer. */ + _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), drflac__mm_packs_interleaved_epi32(left, right)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int16)((pInputSamples0U32[i] << shift0) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)((pInputSamples1U32[i] << shift1) >> 16); + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_independent_stereo__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + int32x4_t shift0_4 = vdupq_n_s32(shift0); + int32x4_t shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + int32x4_t left; + int32x4_t right; + + left = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4)); + right = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4)); + + left = vshrq_n_s32(left, 16); + right = vshrq_n_s32(right, 16); + + drflac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right))); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int16)((pInputSamples0U32[i] << shift0) >> 16); + pOutputSamples[i*2+1] = (drflac_int16)((pInputSamples1U32[i] << shift1) >> 16); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_s16__decode_independent_stereo(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, drflac_int16* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_s16__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_s16__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_s16__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_s16(drflac* pFlac, drflac_uint64 framesToRead, drflac_int16* pBufferOut) +{ + drflac_uint64 framesRead; + drflac_uint32 unusedBitsPerSample; + + if (pFlac == NULL || framesToRead == 0) { + return 0; + } + + if (pBufferOut == NULL) { + return drflac__seek_forward_by_pcm_frames(pFlac, framesToRead); + } + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 32); + unusedBitsPerSample = 32 - pFlac->bitsPerSample; + + framesRead = 0; + while (framesToRead > 0) { + /* If we've run out of samples in this frame, go to the next. */ + if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_and_decode_next_flac_frame(pFlac)) { + break; /* Couldn't read the next frame, so just break from the loop and return. */ + } + } else { + unsigned int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + drflac_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining; + drflac_uint64 frameCountThisIteration = framesToRead; + + if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { + frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; + } + + if (channelCount == 2) { + const drflac_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; + const drflac_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; + + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + drflac_read_pcm_frames_s16__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + drflac_read_pcm_frames_s16__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + drflac_read_pcm_frames_s16__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: + default: + { + drflac_read_pcm_frames_s16__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + } + } else { + /* Generic interleaving. */ + drflac_uint64 i; + for (i = 0; i < frameCountThisIteration; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + drflac_int32 sampleS32 = (drflac_int32)((drflac_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + pBufferOut[(i*channelCount)+j] = (drflac_int16)(sampleS32 >> 16); + } + } + } + + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; + pFlac->currentFLACFrame.pcmFramesRemaining -= (drflac_uint32)frameCountThisIteration; + } + } + + return framesRead; +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_left_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 left = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 side = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (float)((drflac_int32)left / 2147483648.0); + pOutputSamples[i*2+1] = (float)((drflac_int32)right / 2147483648.0); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_left_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + float factor = 1 / 2147483648.0; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 left0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 left1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 left2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 left3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 right0 = left0 - side0; + drflac_uint32 right1 = left1 - side1; + drflac_uint32 right2 = left2 - side2; + drflac_uint32 right3 = left3 - side3; + + pOutputSamples[i*8+0] = (drflac_int32)left0 * factor; + pOutputSamples[i*8+1] = (drflac_int32)right0 * factor; + pOutputSamples[i*8+2] = (drflac_int32)left1 * factor; + pOutputSamples[i*8+3] = (drflac_int32)right1 * factor; + pOutputSamples[i*8+4] = (drflac_int32)left2 * factor; + pOutputSamples[i*8+5] = (drflac_int32)right2 * factor; + pOutputSamples[i*8+6] = (drflac_int32)left3 * factor; + pOutputSamples[i*8+7] = (drflac_int32)right3 * factor; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left * factor; + pOutputSamples[i*2+1] = (drflac_int32)right * factor; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_left_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + __m128 factor; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor = _mm_set1_ps(1.0f / 8388608.0f); + + for (i = 0; i < frameCount4; ++i) { + __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i right = _mm_sub_epi32(left, side); + __m128 leftf = _mm_mul_ps(_mm_cvtepi32_ps(left), factor); + __m128 rightf = _mm_mul_ps(_mm_cvtepi32_ps(right), factor); + + _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf)); + _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left / 8388608.0f; + pOutputSamples[i*2+1] = (drflac_int32)right / 8388608.0f; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_left_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + float32x4_t factor4; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor4 = vdupq_n_f32(1.0f / 8388608.0f); + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t left; + uint32x4_t side; + uint32x4_t right; + float32x4_t leftf; + float32x4_t rightf; + + left = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + right = vsubq_u32(left, side); + leftf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(left)), factor4); + rightf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(right)), factor4); + + drflac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 left = pInputSamples0U32[i] << shift0; + drflac_uint32 side = pInputSamples1U32[i] << shift1; + drflac_uint32 right = left - side; + + pOutputSamples[i*2+0] = (drflac_int32)left / 8388608.0f; + pOutputSamples[i*2+1] = (drflac_int32)right / 8388608.0f; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_left_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_f32__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_f32__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_right_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + for (i = 0; i < frameCount; ++i) { + drflac_uint32 side = (drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + drflac_uint32 right = (drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (float)((drflac_int32)left / 2147483648.0); + pOutputSamples[i*2+1] = (float)((drflac_int32)right / 2147483648.0); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_right_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + float factor = 1 / 2147483648.0; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 side0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 side1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 side2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 side3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 right0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 right1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 right2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 right3 = pInputSamples1U32[i*4+3] << shift1; + + drflac_uint32 left0 = right0 + side0; + drflac_uint32 left1 = right1 + side1; + drflac_uint32 left2 = right2 + side2; + drflac_uint32 left3 = right3 + side3; + + pOutputSamples[i*8+0] = (drflac_int32)left0 * factor; + pOutputSamples[i*8+1] = (drflac_int32)right0 * factor; + pOutputSamples[i*8+2] = (drflac_int32)left1 * factor; + pOutputSamples[i*8+3] = (drflac_int32)right1 * factor; + pOutputSamples[i*8+4] = (drflac_int32)left2 * factor; + pOutputSamples[i*8+5] = (drflac_int32)right2 * factor; + pOutputSamples[i*8+6] = (drflac_int32)left3 * factor; + pOutputSamples[i*8+7] = (drflac_int32)right3 * factor; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left * factor; + pOutputSamples[i*2+1] = (drflac_int32)right * factor; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_right_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + __m128 factor; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor = _mm_set1_ps(1.0f / 8388608.0f); + + for (i = 0; i < frameCount4; ++i) { + __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + __m128i left = _mm_add_epi32(right, side); + __m128 leftf = _mm_mul_ps(_mm_cvtepi32_ps(left), factor); + __m128 rightf = _mm_mul_ps(_mm_cvtepi32_ps(right), factor); + + _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf)); + _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left / 8388608.0f; + pOutputSamples[i*2+1] = (drflac_int32)right / 8388608.0f; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_right_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + float32x4_t factor4; + int32x4_t shift0_4; + int32x4_t shift1_4; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor4 = vdupq_n_f32(1.0f / 8388608.0f); + shift0_4 = vdupq_n_s32(shift0); + shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + uint32x4_t side; + uint32x4_t right; + uint32x4_t left; + float32x4_t leftf; + float32x4_t rightf; + + side = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4); + right = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4); + left = vaddq_u32(right, side); + leftf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(left)), factor4); + rightf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(right)), factor4); + + drflac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 side = pInputSamples0U32[i] << shift0; + drflac_uint32 right = pInputSamples1U32[i] << shift1; + drflac_uint32 left = right + side; + + pOutputSamples[i*2+0] = (drflac_int32)left / 8388608.0f; + pOutputSamples[i*2+1] = (drflac_int32)right / 8388608.0f; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_right_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_f32__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_f32__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_mid_side__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + drflac_uint32 mid = (drflac_uint32)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = (drflac_uint32)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (float)((((drflac_int32)(mid + side) >> 1) << (unusedBitsPerSample)) / 2147483648.0); + pOutputSamples[i*2+1] = (float)((((drflac_int32)(mid - side) >> 1) << (unusedBitsPerSample)) / 2147483648.0); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_mid_side__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample; + float factor = 1 / 2147483648.0; + + if (shift > 0) { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = (mid0 + side0) << shift; + temp1L = (mid1 + side1) << shift; + temp2L = (mid2 + side2) << shift; + temp3L = (mid3 + side3) << shift; + + temp0R = (mid0 - side0) << shift; + temp1R = (mid1 - side1) << shift; + temp2R = (mid2 - side2) << shift; + temp3R = (mid3 - side3) << shift; + + pOutputSamples[i*8+0] = (drflac_int32)temp0L * factor; + pOutputSamples[i*8+1] = (drflac_int32)temp0R * factor; + pOutputSamples[i*8+2] = (drflac_int32)temp1L * factor; + pOutputSamples[i*8+3] = (drflac_int32)temp1R * factor; + pOutputSamples[i*8+4] = (drflac_int32)temp2L * factor; + pOutputSamples[i*8+5] = (drflac_int32)temp2R * factor; + pOutputSamples[i*8+6] = (drflac_int32)temp3L * factor; + pOutputSamples[i*8+7] = (drflac_int32)temp3R * factor; + } + } else { + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 temp0L; + drflac_uint32 temp1L; + drflac_uint32 temp2L; + drflac_uint32 temp3L; + drflac_uint32 temp0R; + drflac_uint32 temp1R; + drflac_uint32 temp2R; + drflac_uint32 temp3R; + + drflac_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + + drflac_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + drflac_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid0 = (mid0 << 1) | (side0 & 0x01); + mid1 = (mid1 << 1) | (side1 & 0x01); + mid2 = (mid2 << 1) | (side2 & 0x01); + mid3 = (mid3 << 1) | (side3 & 0x01); + + temp0L = (drflac_uint32)((drflac_int32)(mid0 + side0) >> 1); + temp1L = (drflac_uint32)((drflac_int32)(mid1 + side1) >> 1); + temp2L = (drflac_uint32)((drflac_int32)(mid2 + side2) >> 1); + temp3L = (drflac_uint32)((drflac_int32)(mid3 + side3) >> 1); + + temp0R = (drflac_uint32)((drflac_int32)(mid0 - side0) >> 1); + temp1R = (drflac_uint32)((drflac_int32)(mid1 - side1) >> 1); + temp2R = (drflac_uint32)((drflac_int32)(mid2 - side2) >> 1); + temp3R = (drflac_uint32)((drflac_int32)(mid3 - side3) >> 1); + + pOutputSamples[i*8+0] = (drflac_int32)temp0L * factor; + pOutputSamples[i*8+1] = (drflac_int32)temp0R * factor; + pOutputSamples[i*8+2] = (drflac_int32)temp1L * factor; + pOutputSamples[i*8+3] = (drflac_int32)temp1R * factor; + pOutputSamples[i*8+4] = (drflac_int32)temp2L * factor; + pOutputSamples[i*8+5] = (drflac_int32)temp2R * factor; + pOutputSamples[i*8+6] = (drflac_int32)temp3L * factor; + pOutputSamples[i*8+7] = (drflac_int32)temp3R * factor; + } + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid + side) >> 1) << unusedBitsPerSample) * factor; + pOutputSamples[i*2+1] = (drflac_int32)((drflac_uint32)((drflac_int32)(mid - side) >> 1) << unusedBitsPerSample) * factor; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_mid_side__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample - 8; + float factor; + __m128 factor128; + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor = 1.0f / 8388608.0f; + factor128 = _mm_set1_ps(factor); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i tempL; + __m128i tempR; + __m128 leftf; + __m128 rightf; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + tempL = _mm_srai_epi32(_mm_add_epi32(mid, side), 1); + tempR = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1); + + leftf = _mm_mul_ps(_mm_cvtepi32_ps(tempL), factor128); + rightf = _mm_mul_ps(_mm_cvtepi32_ps(tempR), factor128); + + _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf)); + _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = ((drflac_int32)(mid + side) >> 1) * factor; + pOutputSamples[i*2+1] = ((drflac_int32)(mid - side) >> 1) * factor; + } + } else { + shift -= 1; + for (i = 0; i < frameCount4; ++i) { + __m128i mid; + __m128i side; + __m128i tempL; + __m128i tempR; + __m128 leftf; + __m128 rightf; + + mid = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + mid = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01))); + + tempL = _mm_slli_epi32(_mm_add_epi32(mid, side), shift); + tempR = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift); + + leftf = _mm_mul_ps(_mm_cvtepi32_ps(tempL), factor128); + rightf = _mm_mul_ps(_mm_cvtepi32_ps(tempR), factor128); + + _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf)); + _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((mid + side) << shift) * factor; + pOutputSamples[i*2+1] = (drflac_int32)((mid - side) << shift) * factor; + } + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_mid_side__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift = unusedBitsPerSample - 8; + float factor; + float32x4_t factor4; + int32x4_t shift4; + int32x4_t wbps0_4; /* Wasted Bits Per Sample */ + int32x4_t wbps1_4; /* Wasted Bits Per Sample */ + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 24); + + factor = 1.0f / 8388608.0f; + factor4 = vdupq_n_f32(factor); + wbps0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); + wbps1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); + + if (shift == 0) { + for (i = 0; i < frameCount4; ++i) { + int32x4_t lefti; + int32x4_t righti; + float32x4_t leftf; + float32x4_t rightf; + + uint32x4_t mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbps0_4); + uint32x4_t side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbps1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1))); + + lefti = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1); + righti = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1); + + leftf = vmulq_f32(vcvtq_f32_s32(lefti), factor4); + rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4); + + drflac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = ((drflac_int32)(mid + side) >> 1) * factor; + pOutputSamples[i*2+1] = ((drflac_int32)(mid - side) >> 1) * factor; + } + } else { + shift -= 1; + shift4 = vdupq_n_s32(shift); + for (i = 0; i < frameCount4; ++i) { + uint32x4_t mid; + uint32x4_t side; + int32x4_t lefti; + int32x4_t righti; + float32x4_t leftf; + float32x4_t rightf; + + mid = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbps0_4); + side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbps1_4); + + mid = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1))); + + lefti = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4)); + righti = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4)); + + leftf = vmulq_f32(vcvtq_f32_s32(lefti), factor4); + rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4); + + drflac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + drflac_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + + mid = (mid << 1) | (side & 0x01); + + pOutputSamples[i*2+0] = (drflac_int32)((mid + side) << shift) * factor; + pOutputSamples[i*2+1] = (drflac_int32)((mid - side) << shift) * factor; + } + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_mid_side(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_f32__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_f32__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + +#if 0 +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_independent_stereo__reference(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + for (drflac_uint64 i = 0; i < frameCount; ++i) { + pOutputSamples[i*2+0] = (float)((drflac_int32)((drflac_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample)) / 2147483648.0); + pOutputSamples[i*2+1] = (float)((drflac_int32)((drflac_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample)) / 2147483648.0); + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_independent_stereo__scalar(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + drflac_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + float factor = 1 / 2147483648.0; + + for (i = 0; i < frameCount4; ++i) { + drflac_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0; + drflac_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0; + drflac_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0; + drflac_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0; + + drflac_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1; + drflac_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1; + drflac_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1; + drflac_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1; + + pOutputSamples[i*8+0] = (drflac_int32)tempL0 * factor; + pOutputSamples[i*8+1] = (drflac_int32)tempR0 * factor; + pOutputSamples[i*8+2] = (drflac_int32)tempL1 * factor; + pOutputSamples[i*8+3] = (drflac_int32)tempR1 * factor; + pOutputSamples[i*8+4] = (drflac_int32)tempL2 * factor; + pOutputSamples[i*8+5] = (drflac_int32)tempR2 * factor; + pOutputSamples[i*8+6] = (drflac_int32)tempL3 * factor; + pOutputSamples[i*8+7] = (drflac_int32)tempR3 * factor; + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0) * factor; + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1) * factor; + } +} + +#if defined(DRFLAC_SUPPORT_SSE2) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_independent_stereo__sse2(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + + float factor = 1.0f / 8388608.0f; + __m128 factor128 = _mm_set1_ps(factor); + + for (i = 0; i < frameCount4; ++i) { + __m128i lefti; + __m128i righti; + __m128 leftf; + __m128 rightf; + + lefti = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); + righti = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); + + leftf = _mm_mul_ps(_mm_cvtepi32_ps(lefti), factor128); + rightf = _mm_mul_ps(_mm_cvtepi32_ps(righti), factor128); + + _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf)); + _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0) * factor; + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1) * factor; + } +} +#endif + +#if defined(DRFLAC_SUPPORT_NEON) +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_independent_stereo__neon(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ + drflac_uint64 i; + drflac_uint64 frameCount4 = frameCount >> 2; + const drflac_uint32* pInputSamples0U32 = (const drflac_uint32*)pInputSamples0; + const drflac_uint32* pInputSamples1U32 = (const drflac_uint32*)pInputSamples1; + drflac_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; + drflac_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; + + float factor = 1.0f / 8388608.0f; + float32x4_t factor4 = vdupq_n_f32(factor); + int32x4_t shift0_4 = vdupq_n_s32(shift0); + int32x4_t shift1_4 = vdupq_n_s32(shift1); + + for (i = 0; i < frameCount4; ++i) { + int32x4_t lefti; + int32x4_t righti; + float32x4_t leftf; + float32x4_t rightf; + + lefti = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4)); + righti = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4)); + + leftf = vmulq_f32(vcvtq_f32_s32(lefti), factor4); + rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4); + + drflac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf)); + } + + for (i = (frameCount4 << 2); i < frameCount; ++i) { + pOutputSamples[i*2+0] = (drflac_int32)(pInputSamples0U32[i] << shift0) * factor; + pOutputSamples[i*2+1] = (drflac_int32)(pInputSamples1U32[i] << shift1) * factor; + } +} +#endif + +static DRFLAC_INLINE void drflac_read_pcm_frames_f32__decode_independent_stereo(drflac* pFlac, drflac_uint64 frameCount, drflac_uint32 unusedBitsPerSample, const drflac_int32* pInputSamples0, const drflac_int32* pInputSamples1, float* pOutputSamples) +{ +#if defined(DRFLAC_SUPPORT_SSE2) + if (drflac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#elif defined(DRFLAC_SUPPORT_NEON) + if (drflac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + drflac_read_pcm_frames_f32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); + } else +#endif + { + /* Scalar fallback. */ +#if 0 + drflac_read_pcm_frames_f32__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#else + drflac_read_pcm_frames_f32__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); +#endif + } +} + +DRFLAC_API drflac_uint64 drflac_read_pcm_frames_f32(drflac* pFlac, drflac_uint64 framesToRead, float* pBufferOut) +{ + drflac_uint64 framesRead; + drflac_uint32 unusedBitsPerSample; + + if (pFlac == NULL || framesToRead == 0) { + return 0; + } + + if (pBufferOut == NULL) { + return drflac__seek_forward_by_pcm_frames(pFlac, framesToRead); + } + + DRFLAC_ASSERT(pFlac->bitsPerSample <= 32); + unusedBitsPerSample = 32 - pFlac->bitsPerSample; + + framesRead = 0; + while (framesToRead > 0) { + /* If we've run out of samples in this frame, go to the next. */ + if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { + if (!drflac__read_and_decode_next_flac_frame(pFlac)) { + break; /* Couldn't read the next frame, so just break from the loop and return. */ + } + } else { + unsigned int channelCount = drflac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + drflac_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining; + drflac_uint64 frameCountThisIteration = framesToRead; + + if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { + frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; + } + + if (channelCount == 2) { + const drflac_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; + const drflac_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; + + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case DRFLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + drflac_read_pcm_frames_f32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + drflac_read_pcm_frames_f32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + drflac_read_pcm_frames_f32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + + case DRFLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: + default: + { + drflac_read_pcm_frames_f32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); + } break; + } + } else { + /* Generic interleaving. */ + drflac_uint64 i; + for (i = 0; i < frameCountThisIteration; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + drflac_int32 sampleS32 = (drflac_int32)((drflac_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + pBufferOut[(i*channelCount)+j] = (float)(sampleS32 / 2147483648.0); + } + } + } + + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; + pFlac->currentFLACFrame.pcmFramesRemaining -= (unsigned int)frameCountThisIteration; + } + } + + return framesRead; +} + + +DRFLAC_API drflac_bool32 drflac_seek_to_pcm_frame(drflac* pFlac, drflac_uint64 pcmFrameIndex) +{ + if (pFlac == NULL) { + return DRFLAC_FALSE; + } + + /* Don't do anything if we're already on the seek point. */ + if (pFlac->currentPCMFrame == pcmFrameIndex) { + return DRFLAC_TRUE; + } + + /* + If we don't know where the first frame begins then we can't seek. This will happen when the STREAMINFO block was not present + when the decoder was opened. + */ + if (pFlac->firstFLACFramePosInBytes == 0) { + return DRFLAC_FALSE; + } + + if (pcmFrameIndex == 0) { + pFlac->currentPCMFrame = 0; + return drflac__seek_to_first_frame(pFlac); + } else { + drflac_bool32 wasSuccessful = DRFLAC_FALSE; + drflac_uint64 originalPCMFrame = pFlac->currentPCMFrame; + + /* Clamp the sample to the end. */ + if (pcmFrameIndex > pFlac->totalPCMFrameCount) { + pcmFrameIndex = pFlac->totalPCMFrameCount; + } + + /* If the target sample and the current sample are in the same frame we just move the position forward. */ + if (pcmFrameIndex > pFlac->currentPCMFrame) { + /* Forward. */ + drflac_uint32 offset = (drflac_uint32)(pcmFrameIndex - pFlac->currentPCMFrame); + if (pFlac->currentFLACFrame.pcmFramesRemaining > offset) { + pFlac->currentFLACFrame.pcmFramesRemaining -= offset; + pFlac->currentPCMFrame = pcmFrameIndex; + return DRFLAC_TRUE; + } + } else { + /* Backward. */ + drflac_uint32 offsetAbs = (drflac_uint32)(pFlac->currentPCMFrame - pcmFrameIndex); + drflac_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; + drflac_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining; + if (currentFLACFramePCMFramesConsumed > offsetAbs) { + pFlac->currentFLACFrame.pcmFramesRemaining += offsetAbs; + pFlac->currentPCMFrame = pcmFrameIndex; + return DRFLAC_TRUE; + } + } + + /* + Different techniques depending on encapsulation. Using the native FLAC seektable with Ogg encapsulation is a bit awkward so + we'll instead use Ogg's natural seeking facility. + */ +#ifndef DR_FLAC_NO_OGG + if (pFlac->container == drflac_container_ogg) + { + wasSuccessful = drflac_ogg__seek_to_pcm_frame(pFlac, pcmFrameIndex); + } + else +#endif + { + /* First try seeking via the seek table. If this fails, fall back to a brute force seek which is much slower. */ + if (/*!wasSuccessful && */!pFlac->_noSeekTableSeek) { + wasSuccessful = drflac__seek_to_pcm_frame__seek_table(pFlac, pcmFrameIndex); + } + +#if !defined(DR_FLAC_NO_CRC) + /* Fall back to binary search if seek table seeking fails. This requires the length of the stream to be known. */ + if (!wasSuccessful && !pFlac->_noBinarySearchSeek && pFlac->totalPCMFrameCount > 0) { + wasSuccessful = drflac__seek_to_pcm_frame__binary_search(pFlac, pcmFrameIndex); + } +#endif + + /* Fall back to brute force if all else fails. */ + if (!wasSuccessful && !pFlac->_noBruteForceSeek) { + wasSuccessful = drflac__seek_to_pcm_frame__brute_force(pFlac, pcmFrameIndex); + } + } + + if (wasSuccessful) { + pFlac->currentPCMFrame = pcmFrameIndex; + } else { + /* Seek failed. Try putting the decoder back to it's original state. */ + if (drflac_seek_to_pcm_frame(pFlac, originalPCMFrame) == DRFLAC_FALSE) { + /* Failed to seek back to the original PCM frame. Fall back to 0. */ + drflac_seek_to_pcm_frame(pFlac, 0); + } + } + + return wasSuccessful; + } +} + + + +/* High Level APIs */ + +/* SIZE_MAX */ +#if defined(SIZE_MAX) + #define DRFLAC_SIZE_MAX SIZE_MAX +#else + #if defined(DRFLAC_64BIT) + #define DRFLAC_SIZE_MAX ((drflac_uint64)0xFFFFFFFFFFFFFFFF) + #else + #define DRFLAC_SIZE_MAX 0xFFFFFFFF + #endif +#endif +/* End SIZE_MAX */ + + +/* Using a macro as the definition of the drflac__full_decode_and_close_*() API family. Sue me. */ +#define DRFLAC_DEFINE_FULL_READ_AND_CLOSE(extension, type) \ +static type* drflac__full_read_and_close_ ## extension (drflac* pFlac, unsigned int* channelsOut, unsigned int* sampleRateOut, drflac_uint64* totalPCMFrameCountOut)\ +{ \ + type* pSampleData = NULL; \ + drflac_uint64 totalPCMFrameCount; \ + \ + DRFLAC_ASSERT(pFlac != NULL); \ + \ + totalPCMFrameCount = pFlac->totalPCMFrameCount; \ + \ + if (totalPCMFrameCount == 0) { \ + type buffer[4096]; \ + drflac_uint64 pcmFramesRead; \ + size_t sampleDataBufferSize = sizeof(buffer); \ + \ + pSampleData = (type*)drflac__malloc_from_callbacks(sampleDataBufferSize, &pFlac->allocationCallbacks); \ + if (pSampleData == NULL) { \ + goto on_error; \ + } \ + \ + while ((pcmFramesRead = (drflac_uint64)drflac_read_pcm_frames_##extension(pFlac, sizeof(buffer)/sizeof(buffer[0])/pFlac->channels, buffer)) > 0) { \ + if (((totalPCMFrameCount + pcmFramesRead) * pFlac->channels * sizeof(type)) > sampleDataBufferSize) { \ + type* pNewSampleData; \ + size_t newSampleDataBufferSize; \ + \ + newSampleDataBufferSize = sampleDataBufferSize * 2; \ + pNewSampleData = (type*)drflac__realloc_from_callbacks(pSampleData, newSampleDataBufferSize, sampleDataBufferSize, &pFlac->allocationCallbacks); \ + if (pNewSampleData == NULL) { \ + drflac__free_from_callbacks(pSampleData, &pFlac->allocationCallbacks); \ + goto on_error; \ + } \ + \ + sampleDataBufferSize = newSampleDataBufferSize; \ + pSampleData = pNewSampleData; \ + } \ + \ + DRFLAC_COPY_MEMORY(pSampleData + (totalPCMFrameCount*pFlac->channels), buffer, (size_t)(pcmFramesRead*pFlac->channels*sizeof(type))); \ + totalPCMFrameCount += pcmFramesRead; \ + } \ + \ + /* At this point everything should be decoded, but we just want to fill the unused part buffer with silence - need to \ + protect those ears from random noise! */ \ + DRFLAC_ZERO_MEMORY(pSampleData + (totalPCMFrameCount*pFlac->channels), (size_t)(sampleDataBufferSize - totalPCMFrameCount*pFlac->channels*sizeof(type))); \ + } else { \ + drflac_uint64 dataSize = totalPCMFrameCount*pFlac->channels*sizeof(type); \ + if (dataSize > (drflac_uint64)DRFLAC_SIZE_MAX) { \ + goto on_error; /* The decoded data is too big. */ \ + } \ + \ + pSampleData = (type*)drflac__malloc_from_callbacks((size_t)dataSize, &pFlac->allocationCallbacks); /* <-- Safe cast as per the check above. */ \ + if (pSampleData == NULL) { \ + goto on_error; \ + } \ + \ + totalPCMFrameCount = drflac_read_pcm_frames_##extension(pFlac, pFlac->totalPCMFrameCount, pSampleData); \ + } \ + \ + if (sampleRateOut) *sampleRateOut = pFlac->sampleRate; \ + if (channelsOut) *channelsOut = pFlac->channels; \ + if (totalPCMFrameCountOut) *totalPCMFrameCountOut = totalPCMFrameCount; \ + \ + drflac_close(pFlac); \ + return pSampleData; \ + \ +on_error: \ + drflac_close(pFlac); \ + return NULL; \ +} + +DRFLAC_DEFINE_FULL_READ_AND_CLOSE(s32, drflac_int32) +DRFLAC_DEFINE_FULL_READ_AND_CLOSE(s16, drflac_int16) +DRFLAC_DEFINE_FULL_READ_AND_CLOSE(f32, float) + +DRFLAC_API drflac_int32* drflac_open_and_read_pcm_frames_s32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, drflac_uint64* totalPCMFrameCountOut, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (channelsOut) { + *channelsOut = 0; + } + if (sampleRateOut) { + *sampleRateOut = 0; + } + if (totalPCMFrameCountOut) { + *totalPCMFrameCountOut = 0; + } + + pFlac = drflac_open(onRead, onSeek, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s32(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut); +} + +DRFLAC_API drflac_int16* drflac_open_and_read_pcm_frames_s16(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, drflac_uint64* totalPCMFrameCountOut, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (channelsOut) { + *channelsOut = 0; + } + if (sampleRateOut) { + *sampleRateOut = 0; + } + if (totalPCMFrameCountOut) { + *totalPCMFrameCountOut = 0; + } + + pFlac = drflac_open(onRead, onSeek, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s16(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut); +} + +DRFLAC_API float* drflac_open_and_read_pcm_frames_f32(drflac_read_proc onRead, drflac_seek_proc onSeek, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, drflac_uint64* totalPCMFrameCountOut, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (channelsOut) { + *channelsOut = 0; + } + if (sampleRateOut) { + *sampleRateOut = 0; + } + if (totalPCMFrameCountOut) { + *totalPCMFrameCountOut = 0; + } + + pFlac = drflac_open(onRead, onSeek, pUserData, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_f32(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut); +} + +#ifndef DR_FLAC_NO_STDIO +DRFLAC_API drflac_int32* drflac_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_file(filename, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s32(pFlac, channels, sampleRate, totalPCMFrameCount); +} + +DRFLAC_API drflac_int16* drflac_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_file(filename, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s16(pFlac, channels, sampleRate, totalPCMFrameCount); +} + +DRFLAC_API float* drflac_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_file(filename, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_f32(pFlac, channels, sampleRate, totalPCMFrameCount); +} +#endif + +DRFLAC_API drflac_int32* drflac_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_memory(data, dataSize, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s32(pFlac, channels, sampleRate, totalPCMFrameCount); +} + +DRFLAC_API drflac_int16* drflac_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_memory(data, dataSize, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_s16(pFlac, channels, sampleRate, totalPCMFrameCount); +} + +DRFLAC_API float* drflac_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, drflac_uint64* totalPCMFrameCount, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + drflac* pFlac; + + if (sampleRate) { + *sampleRate = 0; + } + if (channels) { + *channels = 0; + } + if (totalPCMFrameCount) { + *totalPCMFrameCount = 0; + } + + pFlac = drflac_open_memory(data, dataSize, pAllocationCallbacks); + if (pFlac == NULL) { + return NULL; + } + + return drflac__full_read_and_close_f32(pFlac, channels, sampleRate, totalPCMFrameCount); +} + + +DRFLAC_API void drflac_free(void* p, const drflac_allocation_callbacks* pAllocationCallbacks) +{ + if (pAllocationCallbacks != NULL) { + drflac__free_from_callbacks(p, pAllocationCallbacks); + } else { + drflac__free_default(p, NULL); + } +} + + + + +DRFLAC_API void drflac_init_vorbis_comment_iterator(drflac_vorbis_comment_iterator* pIter, drflac_uint32 commentCount, const void* pComments) +{ + if (pIter == NULL) { + return; + } + + pIter->countRemaining = commentCount; + pIter->pRunningData = (const char*)pComments; +} + +DRFLAC_API const char* drflac_next_vorbis_comment(drflac_vorbis_comment_iterator* pIter, drflac_uint32* pCommentLengthOut) +{ + drflac_int32 length; + const char* pComment; + + /* Safety. */ + if (pCommentLengthOut) { + *pCommentLengthOut = 0; + } + + if (pIter == NULL || pIter->countRemaining == 0 || pIter->pRunningData == NULL) { + return NULL; + } + + length = drflac__le2host_32_ptr_unaligned(pIter->pRunningData); + pIter->pRunningData += 4; + + pComment = pIter->pRunningData; + pIter->pRunningData += length; + pIter->countRemaining -= 1; + + if (pCommentLengthOut) { + *pCommentLengthOut = length; + } + + return pComment; +} + + + + +DRFLAC_API void drflac_init_cuesheet_track_iterator(drflac_cuesheet_track_iterator* pIter, drflac_uint32 trackCount, const void* pTrackData) +{ + if (pIter == NULL) { + return; + } + + pIter->countRemaining = trackCount; + pIter->pRunningData = (const char*)pTrackData; +} + +DRFLAC_API drflac_bool32 drflac_next_cuesheet_track(drflac_cuesheet_track_iterator* pIter, drflac_cuesheet_track* pCuesheetTrack) +{ + drflac_cuesheet_track cuesheetTrack; + const char* pRunningData; + drflac_uint64 offsetHi; + drflac_uint64 offsetLo; + + if (pIter == NULL || pIter->countRemaining == 0 || pIter->pRunningData == NULL) { + return DRFLAC_FALSE; + } + + pRunningData = pIter->pRunningData; + + offsetHi = drflac__be2host_32(*(const drflac_uint32*)pRunningData); pRunningData += 4; + offsetLo = drflac__be2host_32(*(const drflac_uint32*)pRunningData); pRunningData += 4; + cuesheetTrack.offset = offsetLo | (offsetHi << 32); + cuesheetTrack.trackNumber = pRunningData[0]; pRunningData += 1; + DRFLAC_COPY_MEMORY(cuesheetTrack.ISRC, pRunningData, sizeof(cuesheetTrack.ISRC)); pRunningData += 12; + cuesheetTrack.isAudio = (pRunningData[0] & 0x80) != 0; + cuesheetTrack.preEmphasis = (pRunningData[0] & 0x40) != 0; pRunningData += 14; + cuesheetTrack.indexCount = pRunningData[0]; pRunningData += 1; + cuesheetTrack.pIndexPoints = (const drflac_cuesheet_track_index*)pRunningData; pRunningData += cuesheetTrack.indexCount * sizeof(drflac_cuesheet_track_index); + + pIter->pRunningData = pRunningData; + pIter->countRemaining -= 1; + + if (pCuesheetTrack) { + *pCuesheetTrack = cuesheetTrack; + } + + return DRFLAC_TRUE; +} + +#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))) + #pragma GCC diagnostic pop +#endif +#endif /* dr_flac_c */ +#endif /* DR_FLAC_IMPLEMENTATION */ + + +/* +REVISION HISTORY +================ +v0.12.42 - 2023-11-02 + - Fix build for ARMv6-M. + - Fix a compilation warning with GCC. + +v0.12.41 - 2023-06-17 + - Fix an incorrect date in revision history. No functional change. + +v0.12.40 - 2023-05-22 + - Minor code restructure. No functional change. + +v0.12.39 - 2022-09-17 + - Fix compilation with DJGPP. + - Fix compilation error with Visual Studio 2019 and the ARM build. + - Fix an error with SSE 4.1 detection. + - Add support for disabling wchar_t with DR_WAV_NO_WCHAR. + - Improve compatibility with compilers which lack support for explicit struct packing. + - Improve compatibility with low-end and embedded hardware by reducing the amount of stack + allocation when loading an Ogg encapsulated file. + +v0.12.38 - 2022-04-10 + - Fix compilation error on older versions of GCC. + +v0.12.37 - 2022-02-12 + - Improve ARM detection. + +v0.12.36 - 2022-02-07 + - Fix a compilation error with the ARM build. + +v0.12.35 - 2022-02-06 + - Fix a bug due to underestimating the amount of precision required for the prediction stage. + - Fix some bugs found from fuzz testing. + +v0.12.34 - 2022-01-07 + - Fix some misalignment bugs when reading metadata. + +v0.12.33 - 2021-12-22 + - Fix a bug with seeking when the seek table does not start at PCM frame 0. + +v0.12.32 - 2021-12-11 + - Fix a warning with Clang. + +v0.12.31 - 2021-08-16 + - Silence some warnings. + +v0.12.30 - 2021-07-31 + - Fix platform detection for ARM64. + +v0.12.29 - 2021-04-02 + - Fix a bug where the running PCM frame index is set to an invalid value when over-seeking. + - Fix a decoding error due to an incorrect validation check. + +v0.12.28 - 2021-02-21 + - Fix a warning due to referencing _MSC_VER when it is undefined. + +v0.12.27 - 2021-01-31 + - Fix a static analysis warning. + +v0.12.26 - 2021-01-17 + - Fix a compilation warning due to _BSD_SOURCE being deprecated. + +v0.12.25 - 2020-12-26 + - Update documentation. + +v0.12.24 - 2020-11-29 + - Fix ARM64/NEON detection when compiling with MSVC. + +v0.12.23 - 2020-11-21 + - Fix compilation with OpenWatcom. + +v0.12.22 - 2020-11-01 + - Fix an error with the previous release. + +v0.12.21 - 2020-11-01 + - Fix a possible deadlock when seeking. + - Improve compiler support for older versions of GCC. + +v0.12.20 - 2020-09-08 + - Fix a compilation error on older compilers. + +v0.12.19 - 2020-08-30 + - Fix a bug due to an undefined 32-bit shift. + +v0.12.18 - 2020-08-14 + - Fix a crash when compiling with clang-cl. + +v0.12.17 - 2020-08-02 + - Simplify sized types. + +v0.12.16 - 2020-07-25 + - Fix a compilation warning. + +v0.12.15 - 2020-07-06 + - Check for negative LPC shifts and return an error. + +v0.12.14 - 2020-06-23 + - Add include guard for the implementation section. + +v0.12.13 - 2020-05-16 + - Add compile-time and run-time version querying. + - DRFLAC_VERSION_MINOR + - DRFLAC_VERSION_MAJOR + - DRFLAC_VERSION_REVISION + - DRFLAC_VERSION_STRING + - drflac_version() + - drflac_version_string() + +v0.12.12 - 2020-04-30 + - Fix compilation errors with VC6. + +v0.12.11 - 2020-04-19 + - Fix some pedantic warnings. + - Fix some undefined behaviour warnings. + +v0.12.10 - 2020-04-10 + - Fix some bugs when trying to seek with an invalid seek table. + +v0.12.9 - 2020-04-05 + - Fix warnings. + +v0.12.8 - 2020-04-04 + - Add drflac_open_file_w() and drflac_open_file_with_metadata_w(). + - Fix some static analysis warnings. + - Minor documentation updates. + +v0.12.7 - 2020-03-14 + - Fix compilation errors with VC6. + +v0.12.6 - 2020-03-07 + - Fix compilation error with Visual Studio .NET 2003. + +v0.12.5 - 2020-01-30 + - Silence some static analysis warnings. + +v0.12.4 - 2020-01-29 + - Silence some static analysis warnings. + +v0.12.3 - 2019-12-02 + - Fix some warnings when compiling with GCC and the -Og flag. + - Fix a crash in out-of-memory situations. + - Fix potential integer overflow bug. + - Fix some static analysis warnings. + - Fix a possible crash when using custom memory allocators without a custom realloc() implementation. + - Fix a bug with binary search seeking where the bits per sample is not a multiple of 8. + +v0.12.2 - 2019-10-07 + - Internal code clean up. + +v0.12.1 - 2019-09-29 + - Fix some Clang Static Analyzer warnings. + - Fix an unused variable warning. + +v0.12.0 - 2019-09-23 + - API CHANGE: Add support for user defined memory allocation routines. This system allows the program to specify their own memory allocation + routines with a user data pointer for client-specific contextual data. This adds an extra parameter to the end of the following APIs: + - drflac_open() + - drflac_open_relaxed() + - drflac_open_with_metadata() + - drflac_open_with_metadata_relaxed() + - drflac_open_file() + - drflac_open_file_with_metadata() + - drflac_open_memory() + - drflac_open_memory_with_metadata() + - drflac_open_and_read_pcm_frames_s32() + - drflac_open_and_read_pcm_frames_s16() + - drflac_open_and_read_pcm_frames_f32() + - drflac_open_file_and_read_pcm_frames_s32() + - drflac_open_file_and_read_pcm_frames_s16() + - drflac_open_file_and_read_pcm_frames_f32() + - drflac_open_memory_and_read_pcm_frames_s32() + - drflac_open_memory_and_read_pcm_frames_s16() + - drflac_open_memory_and_read_pcm_frames_f32() + Set this extra parameter to NULL to use defaults which is the same as the previous behaviour. Setting this NULL will use + DRFLAC_MALLOC, DRFLAC_REALLOC and DRFLAC_FREE. + - Remove deprecated APIs: + - drflac_read_s32() + - drflac_read_s16() + - drflac_read_f32() + - drflac_seek_to_sample() + - drflac_open_and_decode_s32() + - drflac_open_and_decode_s16() + - drflac_open_and_decode_f32() + - drflac_open_and_decode_file_s32() + - drflac_open_and_decode_file_s16() + - drflac_open_and_decode_file_f32() + - drflac_open_and_decode_memory_s32() + - drflac_open_and_decode_memory_s16() + - drflac_open_and_decode_memory_f32() + - Remove drflac.totalSampleCount which is now replaced with drflac.totalPCMFrameCount. You can emulate drflac.totalSampleCount + by doing pFlac->totalPCMFrameCount*pFlac->channels. + - Rename drflac.currentFrame to drflac.currentFLACFrame to remove ambiguity with PCM frames. + - Fix errors when seeking to the end of a stream. + - Optimizations to seeking. + - SSE improvements and optimizations. + - ARM NEON optimizations. + - Optimizations to drflac_read_pcm_frames_s16(). + - Optimizations to drflac_read_pcm_frames_s32(). + +v0.11.10 - 2019-06-26 + - Fix a compiler error. + +v0.11.9 - 2019-06-16 + - Silence some ThreadSanitizer warnings. + +v0.11.8 - 2019-05-21 + - Fix warnings. + +v0.11.7 - 2019-05-06 + - C89 fixes. + +v0.11.6 - 2019-05-05 + - Add support for C89. + - Fix a compiler warning when CRC is disabled. + - Change license to choice of public domain or MIT-0. + +v0.11.5 - 2019-04-19 + - Fix a compiler error with GCC. + +v0.11.4 - 2019-04-17 + - Fix some warnings with GCC when compiling with -std=c99. + +v0.11.3 - 2019-04-07 + - Silence warnings with GCC. + +v0.11.2 - 2019-03-10 + - Fix a warning. + +v0.11.1 - 2019-02-17 + - Fix a potential bug with seeking. + +v0.11.0 - 2018-12-16 + - API CHANGE: Deprecated drflac_read_s32(), drflac_read_s16() and drflac_read_f32() and replaced them with + drflac_read_pcm_frames_s32(), drflac_read_pcm_frames_s16() and drflac_read_pcm_frames_f32(). The new APIs take + and return PCM frame counts instead of sample counts. To upgrade you will need to change the input count by + dividing it by the channel count, and then do the same with the return value. + - API_CHANGE: Deprecated drflac_seek_to_sample() and replaced with drflac_seek_to_pcm_frame(). Same rules as + the changes to drflac_read_*() apply. + - API CHANGE: Deprecated drflac_open_and_decode_*() and replaced with drflac_open_*_and_read_*(). Same rules as + the changes to drflac_read_*() apply. + - Optimizations. + +v0.10.0 - 2018-09-11 + - Remove the DR_FLAC_NO_WIN32_IO option and the Win32 file IO functionality. If you need to use Win32 file IO you + need to do it yourself via the callback API. + - Fix the clang build. + - Fix undefined behavior. + - Fix errors with CUESHEET metdata blocks. + - Add an API for iterating over each cuesheet track in the CUESHEET metadata block. This works the same way as the + Vorbis comment API. + - Other miscellaneous bug fixes, mostly relating to invalid FLAC streams. + - Minor optimizations. + +v0.9.11 - 2018-08-29 + - Fix a bug with sample reconstruction. + +v0.9.10 - 2018-08-07 + - Improve 64-bit detection. + +v0.9.9 - 2018-08-05 + - Fix C++ build on older versions of GCC. + +v0.9.8 - 2018-07-24 + - Fix compilation errors. + +v0.9.7 - 2018-07-05 + - Fix a warning. + +v0.9.6 - 2018-06-29 + - Fix some typos. + +v0.9.5 - 2018-06-23 + - Fix some warnings. + +v0.9.4 - 2018-06-14 + - Optimizations to seeking. + - Clean up. + +v0.9.3 - 2018-05-22 + - Bug fix. + +v0.9.2 - 2018-05-12 + - Fix a compilation error due to a missing break statement. + +v0.9.1 - 2018-04-29 + - Fix compilation error with Clang. + +v0.9 - 2018-04-24 + - Fix Clang build. + - Start using major.minor.revision versioning. + +v0.8g - 2018-04-19 + - Fix build on non-x86/x64 architectures. + +v0.8f - 2018-02-02 + - Stop pretending to support changing rate/channels mid stream. + +v0.8e - 2018-02-01 + - Fix a crash when the block size of a frame is larger than the maximum block size defined by the FLAC stream. + - Fix a crash the the Rice partition order is invalid. + +v0.8d - 2017-09-22 + - Add support for decoding streams with ID3 tags. ID3 tags are just skipped. + +v0.8c - 2017-09-07 + - Fix warning on non-x86/x64 architectures. + +v0.8b - 2017-08-19 + - Fix build on non-x86/x64 architectures. + +v0.8a - 2017-08-13 + - A small optimization for the Clang build. + +v0.8 - 2017-08-12 + - API CHANGE: Rename dr_* types to drflac_*. + - Optimizations. This brings dr_flac back to about the same class of efficiency as the reference implementation. + - Add support for custom implementations of malloc(), realloc(), etc. + - Add CRC checking to Ogg encapsulated streams. + - Fix VC++ 6 build. This is only for the C++ compiler. The C compiler is not currently supported. + - Bug fixes. + +v0.7 - 2017-07-23 + - Add support for opening a stream without a header block. To do this, use drflac_open_relaxed() / drflac_open_with_metadata_relaxed(). + +v0.6 - 2017-07-22 + - Add support for recovering from invalid frames. With this change, dr_flac will simply skip over invalid frames as if they + never existed. Frames are checked against their sync code, the CRC-8 of the frame header and the CRC-16 of the whole frame. + +v0.5 - 2017-07-16 + - Fix typos. + - Change drflac_bool* types to unsigned. + - Add CRC checking. This makes dr_flac slower, but can be disabled with #define DR_FLAC_NO_CRC. + +v0.4f - 2017-03-10 + - Fix a couple of bugs with the bitstreaming code. + +v0.4e - 2017-02-17 + - Fix some warnings. + +v0.4d - 2016-12-26 + - Add support for 32-bit floating-point PCM decoding. + - Use drflac_int* and drflac_uint* sized types to improve compiler support. + - Minor improvements to documentation. + +v0.4c - 2016-12-26 + - Add support for signed 16-bit integer PCM decoding. + +v0.4b - 2016-10-23 + - A minor change to drflac_bool8 and drflac_bool32 types. + +v0.4a - 2016-10-11 + - Rename drBool32 to drflac_bool32 for styling consistency. + +v0.4 - 2016-09-29 + - API/ABI CHANGE: Use fixed size 32-bit booleans instead of the built-in bool type. + - API CHANGE: Rename drflac_open_and_decode*() to drflac_open_and_decode*_s32(). + - API CHANGE: Swap the order of "channels" and "sampleRate" parameters in drflac_open_and_decode*(). Rationale for this is to + keep it consistent with drflac_audio. + +v0.3f - 2016-09-21 + - Fix a warning with GCC. + +v0.3e - 2016-09-18 + - Fixed a bug where GCC 4.3+ was not getting properly identified. + - Fixed a few typos. + - Changed date formats to ISO 8601 (YYYY-MM-DD). + +v0.3d - 2016-06-11 + - Minor clean up. + +v0.3c - 2016-05-28 + - Fixed compilation error. + +v0.3b - 2016-05-16 + - Fixed Linux/GCC build. + - Updated documentation. + +v0.3a - 2016-05-15 + - Minor fixes to documentation. + +v0.3 - 2016-05-11 + - Optimizations. Now at about parity with the reference implementation on 32-bit builds. + - Lots of clean up. + +v0.2b - 2016-05-10 + - Bug fixes. + +v0.2a - 2016-05-10 + - Made drflac_open_and_decode() more robust. + - Removed an unused debugging variable + +v0.2 - 2016-05-09 + - Added support for Ogg encapsulation. + - API CHANGE. Have the onSeek callback take a third argument which specifies whether or not the seek + should be relative to the start or the current position. Also changes the seeking rules such that + seeking offsets will never be negative. + - Have drflac_open_and_decode() fail gracefully if the stream has an unknown total sample count. + +v0.1b - 2016-05-07 + - Properly close the file handle in drflac_open_file() and family when the decoder fails to initialize. + - Removed a stale comment. + +v0.1a - 2016-05-05 + - Minor formatting changes. + - Fixed a warning on the GCC build. + +v0.1 - 2016-05-03 + - Initial versioned release. +*/ + +/* +This software is available as a choice of the following licenses. Choose +whichever you prefer. + +=============================================================================== +ALTERNATIVE 1 - Public Domain (www.unlicense.org) +=============================================================================== +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +=============================================================================== +ALTERNATIVE 2 - MIT No Attribution +=============================================================================== +Copyright 2023 David Reid + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ diff --git a/deps/zstd/CHANGELOG b/deps/zstd/CHANGELOG new file mode 100644 index 0000000000..c7a7506ee3 --- /dev/null +++ b/deps/zstd/CHANGELOG @@ -0,0 +1,800 @@ +v1.5.5 (Apr 2023) +fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln) +perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln) +lib: deprecated bufferless block-level API (#3534) by @terrelln +cli: mmap large dictionaries to save memory, by @daniellerozenblit +cli: improve speed of --patch-from mode (~+50%) (#3545) by @daniellerozenblit +cli: improve i/o speed (~+10%) when processing lots of small files (#3479) by @felixhandte +cli: zstd no longer crashes when requested to write into write-protected directory (#3541) by @felixhandte +cli: fix decompression into block device using -o, reported by @georgmu (#3583) +build: fix zstd CLI compiled with lzma support but not zlib support (#3494) by @Hello71 +build: fix cmake does no longer require 3.18 as minimum version (#3510) by @kou +build: fix MSVC+ClangCL linking issue (#3569) by @tru +build: fix zstd-dll, version of zstd CLI that links to the dynamic library (#3496) by @yoniko +build: fix MSVC warnings (#3495) by @embg +doc: updated zstd specification to clarify corner cases, by @Cyan4973 +doc: document how to create fat binaries for macos (#3568) by @rickmark +misc: improve seekable format ingestion speed (~+100%) for very small chunk sizes (#3544) by @Cyan4973 +misc: tests/fullbench can benchmark multiple files (#3516) by @dloidolt + +v1.5.4 (Feb 2023) +perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln) +perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg) +perf: +4-13% for levels 5-12 by optimizing function generation (#3295, @terrelln) +pref: +3-11% compression speed for `arm` target (#3199, #3164, #3145, #3141, #3138, @JunHe77 and #3139, #3160, @danlark1) +perf: +5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg) +perf: +10-20% cold dict compression speed by prefetching CDict tables (#3177, @embg) +perf: +1% faster compression by removing a branch in ZSTD_fast_noDict (#3129, @felixhandte) +perf: Small compression ratio improvements in high compression mode (#2983, #3391, @Cyan4973 and #3285, #3302, @daniellerozenblit) +perf: small speed improvement by better detecting `STATIC_BMI2` for `clang` (#3080, @TocarIP) +perf: Improved streaming performance when `ZSTD_c_stableInBuffer` is set (#2974, @Cyan4973) +cli: Asynchronous I/O for improved cli speed (#2975, #2985, #3021, #3022, @yoniko) +cli: Change `zstdless` behavior to align with `zless` (#2909, @binhdvo) +cli: Keep original file if `-c` or `--stdout` is given (#3052, @dirkmueller) +cli: Keep original files when result is concatenated into a single output with `-o` (#3450, @Cyan4973) +cli: Preserve Permissions and Ownership of regular files (#3432, @felixhandte) +cli: Print zlib/lz4/lzma library versions with `-vv` (#3030, @terrelln) +cli: Print checksum value for single frame files with `-lv` (#3332, @Cyan4973) +cli: Print `dictID` when present with `-lv` (#3184, @htnhan) +cli: when `stderr` is *not* the console, disable status updates, but preserve final summary (#3458, @Cyan4973) +cli: support `--best` and `--no-name` in `gzip` compatibility mode (#3059, @dirkmueller) +cli: support for `posix` high resolution timer `clock_gettime()`, for improved benchmark accuracy (#3423, @Cyan4973) +cli: improved help/usage (`-h`, `-H`) formatting (#3094, @dirkmueller and #3385, @jonpalmisc) +cli: Fix better handling of bogus numeric values (#3268, @ctkhanhly) +cli: Fix input consists of multiple files _and_ `stdin` (#3222, @yoniko) +cli: Fix tiny files passthrough (#3215, @cgbur) +cli: Fix for `-r` on empty directory (#3027, @brailovich) +cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg) +cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing) +cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln) +cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln) +api: Support for Block-Level Sequence Producer (#3333, @embg) +api: Support for in-place decompression (#3432, @terrelln) +api: New `ZSTD_CCtx_setCParams()` function, set all parameters defined in a `ZSTD_compressionParameters` structure (#3403, @Cyan4973) +api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973) +api: Window size resizing optimization for edge case (#3345, @daniellerozenblit) +api: More accurate error codes for busy-loop scenarios (#3413, #3455, @Cyan4973) +api: Fix limit overflow in `compressBound` and `decompressBound` (#3362, #3373, Cyan4973) reported by @nigeltao +api: Deprecate several advanced experimental functions: streaming (#3408, @embg), copy (#3196, @mileshu) +bug: Fix corruption that rarely occurs in 32-bit mode with wlog=25 (#3361, @terrelln) +bug: Fix for block-splitter (#3033, @Cyan4973) +bug: Fixes for Sequence Compression API (#3023, #3040, @Cyan4973) +bug: Fix leaking thread handles on Windows (#3147, @animalize) +bug: Fix timing issues with cmake/meson builds (#3166, #3167, #3170, @Cyan4973) +build: Allow user to select legacy level for cmake (#3050, @shadchin) +build: Enable legacy support by default in cmake (#3079, @niamster) +build: Meson build script improvements (#3039, #3120, #3122, #3327, #3357, @eli-schwartz and #3276, @neheb) +build: Add aarch64 to supported architectures for zstd_trace (#3054, @ooosssososos) +build: support AIX architecture (#3219, @qiongsiwu) +build: Fix `ZSTD_LIB_MINIFY` build macro, which now reduces static library size by half (#3366, @terrelln) +build: Fix Windows issues with Multithreading translation layer (#3364, #3380, @yoniko) and ARM64 target (#3320, @cwoffenden) +build: Fix `cmake` script (#3382, #3392, @terrelln and #3252 @Tachi107 and #3167 @Cyan4973) +doc: Updated man page, providing more details for `--train` mode (#3112, @Cyan4973) +doc: Add decompressor errata document (#3092, @terrelln) +misc: Enable Intel CET (#2992, #2994, @hjl-tools) +misc: Fix `contrib/` seekable format (#3058, @yhoogstrate and #3346, @daniellerozenblit) +misc: Improve speed of the one-file library generator (#3241, @wahern and #3005, @cwoffenden) + +v1.5.3 (dev version, unpublished) + +v1.5.2 (Jan, 2022) +perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969) +build: Build Zstd with `noexecstack` on All Architectures (@felixhandte, #2964) +doc: Clarify Licensing (@terrelln, #2981) + +v1.5.1 (Dec, 2021) +perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42 +perf: faster huffman decoder, using x64 assembly, by @terrelln +perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte +perf: improved binary size and faster compilation times, by @terrelln +perf: new row64 mode, used notably in level 12, by @senhuang42 +perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42 +perf: minor compression ratio improvements for small data at high levels, by @cyan4973 +perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln +perf: faster compression speed on incompressible data, by @bindhvo +perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo +build: allows hiding static symbols in the dynamic library, using build macro, by @skitt +build: support for m68k (Motorola 68000's), by @cyan4973 +build: improved AIX support, by @Helflym +build: improved meson unofficial build, by @eli-schwartz +cli : custom memory limit when training dictionary (#2925), by @embg +cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB + +v1.5.0 (May 11, 2021) +api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42) + `ZSTD_defaultCLevel()` + `ZSTD_getDictID_fromCDict()` +api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42) + `ZSTD_compress_advanced()` + `ZSTD_compress_usingCDict_advanced()` + `ZSTD_compressBegin_advanced()` + `ZSTD_compressBegin_usingCDict_advanced()` + `ZSTD_initCStream_srcSize()` + `ZSTD_initCStream_usingDict()` + `ZSTD_initCStream_usingCDict()` + `ZSTD_initCStream_advanced()` + `ZSTD_initCStream_usingCDict_advanced()` + `ZSTD_resetCStream()` +api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973) +perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln) +perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42) +perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973) +perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln) +bug: Improve setting permissions of created files (#2525, @felixhandte) +bug: Fix large dictionary non-determinism (#2607, @terrelln) +bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln) +bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte) +bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42) +bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink) +bug: Fix UBSAN error in decompression (#2625, @terrelln) +bug: Fix superblock compression divide by zero bug (#2592, @senhuang42) +bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone) +doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln) +doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize) +doc: Add style guide docs for open source contributors (#2626, @Cyan4973) +tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42) +tests: Better test coverage of index reduction (#2603, @terrelln) +tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42) +tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42) +build: Dynamic library built multithreaded by default (#2584, @senhuang42) +build: Move `zstd_errors.h` and `zdict.h` to `lib/` root (#2597, @terrelln) +build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973) +build: Single file library build script moved to `build/` directory (#2618, @felixhandte) +build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42) +build: Fixed Meson build (#2548, @SupervisedThinking @kloczek) +build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson) +build: Detect presence of `md5` on Darwin (#2609, @felixhandte) +build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann) +cli: `--progress` flag added to always display progress bar (#2595, @senhuang42) +cli: Allow reading from block devices with `--force` (#2613, @felixhandte) +cli: Fix CLI filesize display bug (#2550, @Cyan4973) +cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973) +contrib: Various fixes for linux kernel patch (#2539, @terrelln) +contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42) +contrib: Seekable format - New seek table-only API (#2113 #2518, @mdittmer @Cyan4973) +contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng) +contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat) +misc: Automatically published release tarballs available on Github (#2535, @felixhandte) + +v1.4.9 (Mar 1, 2021) +bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte) +bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln) +api: Add (De)Compression Tracing Functionality (#2482, @terrelln) +api: Support References to Multiple DDicts (#2446, @senhuang42) +api: Add Function to Generate Skippable Frame (#2439, @senhuang42) +perf: New Algorithms for the Long Distance Matcher (#2483, @mpu) +perf: Performance Improvements for Long Distance Matcher (#2464, @mpu) +perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln) +cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte) +cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte) +cli: Improve Help Message (#2500, @senhuang42) +tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973) +tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat) +tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte) +build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973) +build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt) +build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime) +build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka) +build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln) +build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg) +build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973) +contrib: Add `recover_directory` Program (#2473, @terrelln) +doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42) +doc: Fix Typos (#2459, @ThomasWaldmann) + +v1.4.8 (Dec 18, 2020) +hotfix: wrong alignment of an internal buffer + +v1.4.7 (Dec 16, 2020) +perf: stronger --long mode at high compression levels, by @senhuang42 +perf: stronger --patch-from at high compression levels, thanks to --long improvements +perf: faster dictionary compression at medium compression levels, by @felixhandte +perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln +perf: improved fast compression speeds with Visual Studio, by @animalize +cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42 +cli : accept decompressing files with *.zstd suffix +cli : provide a condensed summary by default when processing multiple files +cli : fix : stdin input no longer confused as user prompt +cli : improve accuracy of several error messages +api : new sequence ingestion API, by @senhuang42 +api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin +api : new ZSTD_getDictID_fromCDict(), by @LuAPi +api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln +api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli) +api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli) +build: fix cmake script when using path with spaces, by @terrelln +build: improved compile-time detection of aarch64/neon platforms, by @bsdimp +build: Fix building on AIX 5.1, by @likema +build: compile paramgrill with cmake on Windows, requested by @mirh +doc : clarify repcode updates in format specification, by @felixhandte + +v1.4.6 +fix : Always return dstSize_tooSmall when that is the case +fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary +perf: Improve small block decompression speed by 20%+, by @terrelln +perf: Reduce compression stack usage by 1 KB, by @terrelln +perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256) +perf: Improve histogram construction, by @cyan4973 (#2253) +cli : Add --output-dir-mirror option, by @xxie24 (#2219) +cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279) +cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283) +cli : Call stat less often, by @felixhandte (#2262) +cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250) +cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206) +api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166). +api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291) +api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228). +build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183) +build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221) +build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186) +build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions +build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics +build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258) +build: Fix -Wcomma warnings, by @cwoffenden +build: Remove distutils requirement for meson build, by @neheb (#2197) +build: Fix cli compilation with uclibc +build: Fix cli compilation without st_mtime, by @ffontaine (#2246) +build: Fix shadowing warnings in library +build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227) +misc: Improve single file library and include dictBuilder, by @cwoffenden +misc: Allow compression dictionaries with missing symbols +misc: Add freestanding translation script in contrib/freestanding_lib +misc: Collect all of zstd's libc dependencies into zstd_deps.h +doc : Add ZSTD_versionString() to manual, by @animalize +doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270) + +v1.4.5 (May 22, 2020) +fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln +perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln +perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta) +perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh) +perf: Small level 1 compression speed gains (depending on compiler) +cli : New --patch-from command, create and apply patches from files, by @bimbashreshta +cli : New --filelist= : Provide a list of files to operate upon from a file +cli : -b -d command can now benchmark decompression on multiple files +cli : New --no-content-size command +cli : New --show-default-cparams information command +api : ZDICT_finalizeDictionary() is promoted to stable (#2111) +api : new experimental parameter ZSTD_d_stableOutBuffer (#2094) +build: Generate a single-file libzstd library (#2065, by @cwoffenden) +build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte) +build: zstd now compiles cleanly under -pedantic (#2099) +build: zstd now compiles with make-4.3 +build: Support mingw cross-compilation from Linux, by @Ericson2314 +build: Meson multi-thread build fix on windows +build: Some misc icc fixes backed by new ci test on travis +misc: bitflip analyzer tool, by @felixhandte +misc: Extend largeNbDicts benchmark to compression +misc: Edit-distance match finder in contrib/ +doc : Improved beginner CONTRIBUTING.md docs +doc : New issue templates for zstd + +v1.4.4 (Nov 6, 2019) +perf: Improved decompression speed, by > 10%, by @terrelln +perf: Better compression speed when re-using a context, by @felixhandte +perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42 +perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha +perf: minor generic speed optimization, by @davidbolvansky +api: new ability to extract sequences from the parser for analysis, by @bimbashrestha +api: fixed decoding of magic-less frames, by @terrelln +api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK +cli: Named pipes support, by @bimbashrestha +cli: short tar's extension support, by @stokito +cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42 +cli: commands --stream-size=# and --size-hint=#, by @nmagerko +cli: command --exclude-compressed, by @shashank0791 +cli: faster `-t` test mode +cli: improved some error messages, by @vangyzen +cli: fix command `-D dictionary` on Windows, reported by @artyompetrov +cli: fix rare deadlock condition within dictionary builder, by @terrelln +build: single-file decoder with emscripten compilation script, by @cwoffenden +build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive +build: fixed deprecation warning for certain gcc version, reported by @jasonma163 +build: fix compilation on old gcc versions, by @cemeyer +build: improved installation directories for cmake script, by Dmitri Shubin +pack: modified pkgconfig, for better integration into openwrt, requested by @neheb +misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format +misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro + +v1.4.3 (Aug 20, 2019) +bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709) +bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722) +build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705) + +v1.4.2 (Jul 26, 2019) +bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696) +bug: Fix seekable decompression in-memory API by @iburinoc (#1695) +misc: Validate blocks are smaller than size limit by @vivekmg (#1685) +misc: Restructure source files by @ephiepark (#1679) + +v1.4.1 (Jul 20, 2019) +bug: Fix data corruption in niche use cases by @terrelln (#1659) +bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595) +bug: Fix out of bounds read by @terrelln (#1590) +perf: Improve decode speed by ~7% @mgrice (#1668) +perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681) +perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658) +perf: Improve compression ratio for small windowLog by @cyan4973 (#1624) +perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635) +api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656) +cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640) +cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631) +cli: Restrict read permissions on destination files by @chungy (#1644) +cli: zstdgrep: handle -f flag by @felixhandte (#1618) +cli: zstdcat: follow symlinks by @vejnar (#1604) +doc: Remove extra size limit on compressed blocks by @felixhandte (#1689) +doc: Fix typo by @yk-tanigawa (#1633) +doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629) +build: CMake: support building with LZ4 @leeyoung624 (#1626) +build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647) +build: CMake: respect existing uninstall target by @j301scott (#1619) +build: Make: skip multithread tests when built without support by @michaelforney (#1620) +build: Make: Fix examples/ test target by @sjnam (#1603) +build: Meson: rename options out of deprecated namespace by @lzutao (#1665) +build: Meson: fix build by @lzutao (#1602) +build: Visual Studio: don't export symbols in static lib by @scharan (#1650) +build: Visual Studio: fix linking by @absotively (#1639) +build: Fix MinGW-W64 build by @myzhang1029 (#1600) +misc: Expand decodecorpus coverage by @ephiepark (#1664) + +v1.4.0 (Apr 17, 2019) +perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln +api: Move the advanced API, including all functions in the staging section, to the stable section +api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress +api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter +api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter +api: Don't export ZSTDMT functions from the shared library by default +api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT +api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao +api: Fix ZSTD_decompressDCtx() corner cases with a dictionary +api: Move ZSTD_getDictID_*() functions to the stable section +api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln +api: Allow compression parameters to be set when a dictionary is used +api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called +api: Fix ZSTD_estimateCStreamSize_usingCCtxParams() +api: Setting ZSTD_d_maxWindowLog to 0 means use the default +cli: Ensure that a dictionary is not used to compress itself by @shakeelrao +cli: Add --[no-]compress-literals flag to enable or disable literal compression +doc: Update the examples to use the advanced API +doc: Explain how to transition from old streaming functions to the advanced API in the header +build: Improve the Windows release packages +build: Improve CMake build by @hjmjohnson +build: Build fixes for FreeBSD by @lwhsu +build: Remove redundant warnings by @thatsafunnyname +build: Fix tests on OpenBSD by @bket +build: Extend fuzzer build system to work with the new clang engine +build: CMake now creates the libzstd.so.1 symlink +build: Improve Menson build by @lzutao +misc: Fix symbolic link detection on FreeBSD +misc: Use physical core count for -T0 on FreeBSD by @cemeyer +misc: Fix zstd --list on truncated files by @kostmo +misc: Improve logging in debug mode by @felixhandte +misc: Add CirrusCI tests by @lwhsu +misc: Optimize dictionary memory usage in corner cases +misc: Improve the dictionary builder on small or homogeneous data +misc: Fix spelling across the repo by @jsoref + +v1.3.8 (Dec 28, 2018) +perf: better decompression speed on large files (+7%) and cold dictionaries (+15%) +perf: slightly better compression ratio at high compression modes +api : finalized advanced API, last stage before "stable" status +api : new --rsyncable mode, by @terrelln +api : support decompression of empty frames into NULL (used to be an error) (#1385) +build: new set of macros to build a minimal size decoder, by @felixhandte +build: fix compilation on MIPS32, reported by @clbr (#1441) +build: fix compilation with multiple -arch flags, by @ryandesign +build: highly upgraded meson build, by @lzutao +build: improved buck support, by @obelisk +build: fix cmake script : can create debug build, by @pitrou +build: Makefile : grep works on both colored consoles and systems without color support +build: fixed zstd-pgo, by @bmwiedemann +cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423) +cli : --no-progress flag, preserving final summary (#1371), by @terrelln +cli : ensure destination file is not source file (#1422) +cli : clearer error messages, especially when input file not present +doc : clarified zstd_compression_format.md, by @ulikunitz +misc: fixed zstdgrep, returns 1 on failure, by @lzutao +misc: NEWS renamed as CHANGELOG, in accordance with fboss + +v1.3.7 (Oct 20, 2018) +perf: slightly better decompression speed on clang (depending on hardware target) +fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10 +build: no longer build backtrace by default in release mode; restrict further automatic mode +build: control backtrace support through build macro BACKTRACE +misc: added man pages for zstdless and zstdgrep, by @samrussell + +v1.3.6 (Oct 6, 2018) +perf: much faster dictionary builder, by @jenniferliu +perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte +perf: faster dictionary decompression when using a very large number of dictionaries simultaneously +cli : fix : does no longer overwrite destination when source does not exist (#1082) +cli : new command --adapt, for automatic compression level adaptation +api : fix : block api can be streamed with > 4 GB, reported by @catid +api : reduced ZSTD_DDict size by 2 KB +api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel(). +build: support Haiku target, by @korli +build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT. +doc : zstd_compression_format.md updated to match wording in IETF RFC 8478 +misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97 + +v1.3.5 (Jun 29, 2018) +perf: much faster dictionary compression, by @felixhandte +perf: small quality improvement for dictionary generation, by @terrelln +perf: slightly improved high compression levels (notably level 19) +mem : automatic memory release for long duration contexts +cli : fix : overlapLog can be manually set +cli : fix : decoding invalid lz4 frames +api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln +api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln +build: select custom libzstd scope through control macros, by @GeorgeLu97 +build: OpenBSD patch, by @bket +build: make and make all are compatible with -j +doc : clarify zstd_compression_format.md, updated for IETF RFC process +misc: pzstd compatible with reproducible compilation, by @lamby + +v1.3.4 (Mar 27, 2018) +perf: faster speed (especially decoding speed) on recent cpus (haswell+) +perf: much better performance associating --long with multi-threading, by @terrelln +perf: better compression at levels 13-15 +cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior) +cli : smoother status report in multi-threading mode +cli : added command --fast=#, for faster compression modes +cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb) +api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode +api : compression levels can be negative, for even more speed +api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime +api : ZSTDMT can accept new compression parameters during compression +api : implemented all advanced dictionary decompression prototypes +build: improved meson recipe, by Shawn Landden (@shawnl) +build: VS2017 scripts, by @HaydnTrigg +misc: all /contrib projects fixed +misc: added /contrib/docker script by @gyscos + +v1.3.3 (Dec 21, 2017) +perf: faster zstd_opt strategy (levels 16-19) +fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen +cli : fix : content size written in header by default +cli : fix : improved LZ4 format support, by @felixhandte +cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file +api : fix : support large skippable frames, by @terrelln +api : fix : streaming interface was adding a useless 3-bytes null block to small frames +api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown" +build: fix : compilation under rhel6 and centos6, reported by @pixelb +build: added `check` target + +v1.3.2 (Oct 10, 2017) +new : long range mode, using --long command, by Stella Lau (@stellamplau) +new : ability to generate and decode magicless frames (#591) +changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode +fix : multi-threading compression works with custom allocators +fix : ZSTD_sizeof_CStream() was over-evaluating memory usage +fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22) +fix : 32-bits build can now decode large offsets (levels 21+) +cli : added LZ4 frame support by default, by Felix Handte (@felixhandte) +cli : improved --list output +cli : new : can split input file for dictionary training, using command -B# +cli : new : clean operation artefact on Ctrl-C interruption +cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851) +cli : fix : write file size in header in multiple-files mode +api : added macro ZSTD_COMPRESSBOUND() for static allocation +api : experimental : new advanced decompression API +api : fix : sizeof_CCtx() used to over-estimate +build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819) +build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818) +example : added streaming_memory_usage +license : changed /examples license to BSD + GPLv2 +license : fix a few header files to reflect new license (#825) + +v1.3.1 (Aug 21, 2017) +New license : BSD + GPLv2 +perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) +perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) +cli : improved and fixed --list command, by @ib (#772) +cli : command -vV to list supported formats, by @ib (#771) +build : fixed binary variants, reported by @svenha (#788) +build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718) +API exp : breaking change : ZSTD_getframeHeader() provides more information +API exp : breaking change : pinned down values of error codes +doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz) +new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74) +new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) +updated : contrib/linux-kernel, by Nick Terrell (@terrelln) + +v1.3.0 (Jul 6, 2017) +cli : new : `--list` command, by Paul Cruz +cli : changed : xz/lzma support enabled by default +cli : changed : `-t *` continue processing list after a decompression error +API : added : ZSTD_versionString() +API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell +API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter() +API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx() +API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700) +API exp : clarified memory estimation / measurement functions. +API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1 +tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz +new : contrib/seekable_format, demo and API, by Sean Purcell +changed : contrib/linux-kernel, updated version and license, by Nick Terrell + +v1.2.0 (May 5, 2017) +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + +v1.1.4 (Mar 18, 2017) +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +cli : fix : --rm remains silent when input is stdin +cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() +API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady +doc : new : educational decoder, by Sean Purcell + +v1.1.3 (Feb 7, 2017) +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) +cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. +cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski +cli : fix zstdless on Mac OS-X, by Andrew Janke +cli : fix #232 "compress non-files" +dictBuilder : improved dictionary generation quality, thanks to Nick Terrell +API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul +API : new : ZDICT_finalizeDictionary() +API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) +API : fix : all symbols properly exposed in libzstd, by Nick Terrell +build : support for Solaris target, by Przemyslaw Skibinski +doc : clarified specification, by Sean Purcell + +v1.1.2 (Dec 15, 2016) +API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init +API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated + +v1.1.1 (Nov 2, 2016) +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) +Updated man page +Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets + +v1.1.0 (Sep 28, 2016) +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +added : NetBSD install target (#338) +Improved : speed for batches of small files +Improved : speed of zlib wrapper, by Przemyslaw Skibinski +Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) +Fixed : zstd-pgo, reported by octoploid (#329) + +v1.0.0 (Sep 1, 2016) +Change Licensing, all project is now BSD, Copyright Facebook +Small decompression speed improvement +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew +Enhanced documentation, by Przemyslaw Skibinski + +v0.8.1 (Aug 18, 2016) +New streaming API +Changed : --ultra now enables levels beyond 19 +Changed : -i# now selects benchmark time in second +Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell +Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski + +v0.8.0 (Aug 2, 2016) +Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +New : Build on FreeBSD and DragonFly, thanks to JrMarino +Changed : modified API : ZSTD_compressEnd() +Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist +Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers +Fixed : large dictionaries (> 384 KB), reported by Ilona Papava +Fixed : checksum correctly checked in single-pass mode +Fixed : combined --test amd --rm, reported by Andreas M. Nilsson +Modified : minor compression level adaptations +Updated : compression format specification to v0.2.0 +changed : zstd.h moved to /lib directory + +v0.7.5 (Aug 1, 2016) +Transition version, supporting decoding of v0.8.x + +v0.7.4 (Jul 17, 2016) +Added : homebrew for Mac, by Daniel Cade +Added : more examples +Fixed : segfault when using small dictionaries, reported by Felix Handte +Modified : default compression level for CLI is now 3 +Updated : specification, to v0.1.1 + +v0.7.3 (Jul 9, 2016) +New : compression format specification +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory +fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number + +v0.7.2 (Jul 4, 2016) +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. +fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. +fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. + +v0.7.1 (Jun 23, 2016) +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв +fixed : corruption issue, reported by cj +modified : checksum enabled by default in command line mode + +v0.7.0 (Jun 17, 2016) +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression +New : Visual build scripts, by Christophe Chevalier +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support +New : Support pass-through mode (when using `-df`) +API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` +API : create dictionary files from custom content, by Giuseppe Ottaviano +API : support for custom malloc/free functions +New : controllable Dictionary ID +New : Support for skippable frames + +v0.6.1 (May 13, 2016) +New : zlib wrapper API, thanks to Przemyslaw Skibinski +New : Ability to compile compressor / decompressor separately +Changed : new lib directory structure +Fixed : Legacy codec v0.5 compatible with dictionary decompression +Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) +New : benchmark mode can select directory as input +Experimental : midipix support, VMS support + +v0.6.0 (Apr 13, 2016) +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + +v0.5.1 (Feb 18, 2016) +New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. +Fix : high compression modes for big-endian platforms +New : zstd cli : `-t` | `--test` command + +v0.5.0 (Feb 5, 2016) +New : dictionary builder utility +Changed : streaming & dictionary API +Improved : better compression of small data + +v0.4.7 (Jan 22, 2016) +Improved : small compression speed improvement in HC mode +Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default +fix : bt search bug + +v0.4.6 (Jan 13, 2016) +fix : fast compression mode on Windows +New : cmake configuration file, thanks to Artyom Dymchenko +Improved : high compression mode on repetitive data +New : block-level API +New : ZSTD_duplicateCCtx() + +v0.4.5 (Dec 18, 2015) +new : -m/--multiple : compress/decompress multiple files + +v0.4.4 (Dec 14, 2015) +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + +v0.4.3 (Dec 7, 2015) +new : external dictionary API +new : zstd-frugal + +v0.4.2 (Dec 2, 2015) +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + +v0.4.1 (Dec 1, 2015) +Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) +removed `zstd.c` + +v0.4.0 (Nov 29, 2015) +Command line utility compatible with high compression levels +Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) +Rolling buffer support + +v0.3.6 (Nov 10, 2015) +small blocks params + +v0.3.5 (Nov 9, 2015) +minor generic compression improvements + +v0.3.4 (Nov 6, 2015) +Faster fast cLevels + +v0.3.3 (Nov 5, 2015) +Small compression ratio improvement + +v0.3.2 (Nov 2, 2015) +Fixed Visual Studio + +v0.3.1 (Nov 2, 2015) +Small compression ratio improvement + +v0.3 (Oct 30, 2015) +HC mode : compression levels 2-26 + +v0.2.2 (Oct 28, 2015) +Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier + +v0.2.1 (Oct 24, 2015) +Fix : Read errors, advanced fuzzer tests, by Hanno Böck + +v0.2.0 (Oct 22, 2015) +**Breaking format change** +Faster decompression speed +Can still decode v0.1 format + +v0.1.3 (Oct 15, 2015) +fix uninitialization warning, reported by Evan Nemerson + +v0.1.2 (Sep 11, 2015) +frame concatenation support + +v0.1.1 (Aug 27, 2015) +fix compression bug +detects write-flush errors + +v0.1.0 (Aug 25, 2015) +first release diff --git a/deps/zstd/CODE_OF_CONDUCT.md b/deps/zstd/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..0f7ad8bfc1 --- /dev/null +++ b/deps/zstd/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# Code of Conduct + +Facebook has adopted a Code of Conduct that we expect project participants to adhere to. +Please read the [full text](https://code.fb.com/codeofconduct/) +so that you can understand what actions will and will not be tolerated. diff --git a/deps/zstd/CONTRIBUTING.md b/deps/zstd/CONTRIBUTING.md new file mode 100644 index 0000000000..f5e747ae1b --- /dev/null +++ b/deps/zstd/CONTRIBUTING.md @@ -0,0 +1,489 @@ +# Contributing to Zstandard +We want to make contributing to this project as easy and transparent as +possible. + +## Our Development Process +New versions are being developed in the "dev" branch, +or in their own feature branch. +When they are deemed ready for a release, they are merged into "release". + +As a consequence, all contributions must stage first through "dev" +or their own feature branch. + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `dev`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints. +6. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## Workflow +Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd +will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple +related changes into a branch. + +Our contribution process works in three main stages: +1. Local development + * Update: + * Checkout your fork of zstd if you have not already + ``` + git checkout https://github.com//zstd + cd zstd + ``` + * Update your local dev branch + ``` + git pull https://github.com/facebook/zstd dev + git push origin dev + ``` + * Topic and development: + * Make a new branch on your fork about the topic you're developing for + ``` + # branch names should be concise but sufficiently informative + git checkout -b + git push origin + ``` + * Make commits and push + ``` + # make some changes = + git add -u && git commit -m + git push origin + ``` + * Note: run local tests to ensure that your changes didn't break existing functionality + * Quick check + ``` + make shortest + ``` + * Longer check + ``` + make test + ``` +2. Code Review and CI tests + * Ensure CI tests pass: + * Before sharing anything to the community, create a pull request in your own fork against the dev branch + and make sure that all GitHub Actions CI tests pass. See the Continuous Integration section below for more information. + * Ensure that static analysis passes on your development machine. See the Static Analysis section + below to see how to do this. + * Create a pull request: + * When you are ready to share you changes to the community, create a pull request from your branch + to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home + page. + * From there, select the branch where you made changes as your source branch and facebook:dev + as the destination. + * Examine the diff presented between the two branches to make sure there is nothing unexpected. + * Write a good pull request description: + * While there is no strict template that our contributors follow, we would like them to + sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests, + at least indirectly, address the following points. + * Is this pull request important and why? + * Is it addressing an issue? If so, what issue? (provide links for convenience please) + * Is this a new feature? If so, why is it useful and/or necessary? + * Are there background references and documents that reviewers should be aware of to properly assess this change? + * Note: make sure to point out any design and architectural decisions that you made and the rationale behind them. + * Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@) + * Submit the pull request and iterate with feedback. +3. Merge and Release + * Getting approval: + * You will have to iterate on your changes with feedback from other collaborators to reach a point + where your pull request can be safely merged. + * To avoid too many comments on style and convention, make sure that you have a + look at our style section below before creating a pull request. + * Eventually, someone from the zstd team will approve your pull request and not long after merge it into + the dev branch. + * Housekeeping: + * Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure + the corresponding issue is mentioned. If your change 'fixes' or completely addresses the + issue at hand, then please indicate this by requesting that an issue be closed by commenting. + * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember + that the change must make it to an official zstd release for it to be meaningful. We recommend + that contributors track the activity on their pull request and corresponding issue(s) page(s) until + their change makes it to the next release of zstd. Users will often discover bugs in your code or + suggest ways to refine and improve your initial changes even after the pull request is merged. + +## Static Analysis +Static analysis is a process for examining the correctness or validity of a program without actually +executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for +static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build. + +Once installed, you can ensure that our static analysis tests pass on your local development machine +by running: +``` +make staticAnalyze +``` + +In general, you can use `scan-build` to static analyze any build script. For example, to static analyze +just `contrib/largeNbDicts` and nothing else, you can run: + +``` +scan-build make -C contrib/largeNbDicts largeNbDicts +``` + +### Pitfalls of static analysis +`scan-build` is part of our regular CI suite. Other static analyzers are not. + +It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are : + +- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low. +- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload. +- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to : + + tortuous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers. + + sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time. +- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer. + +This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kinds of reports can be useful, and are accepted. + +## Continuous Integration +CI tests run every time a pull request (PR) is created or updated. The exact tests +that get run will depend on the destination branch you specify. Some tests take +longer to run than others. Currently, our CI is set up to run a short +series of tests when creating a PR to the dev branch and a longer series of tests +when creating a PR to the release branch. You can look in the configuration files +of the respective CI platform for more information on what gets run when. + +Most people will just want to create a PR with the destination set to their local dev +branch of zstd. You can then find the status of the tests on the PR's page. You can also +re-run tests and cancel running tests from the PR page or from the respective CI's dashboard. + +Almost all of zstd's CI runs on GitHub Actions (configured at `.github/workflows`), which will automatically run on PRs to your +own fork. A small number of tests run on other services (e.g. Travis CI, Circle CI, Appveyor). +These require work to set up on your local fork, and (at least for Travis CI) cost money. +Therefore, if the PR on your local fork passes GitHub Actions, feel free to submit a PR +against the main repo. + +### Third-party CI +A small number of tests cannot run on GitHub Actions, or have yet to be migrated. +For these, we use a variety of third-party services (listed below). It is not necessary to set +these up on your fork in order to contribute to zstd; however, we do link to instructions for those +who want earlier signal. + +| Service | Purpose | Setup Links | Config Path | +|-----------|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| Travis CI | Used for testing on non-x86 architectures such as PowerPC | https://docs.travis-ci.com/user/tutorial/#to-get-started-with-travis-ci-using-github
https://github.com/marketplace/travis-ci | `.travis.yml` | +| AppVeyor | Used for some Windows testing (e.g. cygwin, mingw) | https://www.appveyor.com/blog/2018/10/02/github-apps-integration/
https://github.com/marketplace/appveyor | `appveyor.yml` | +| Cirrus CI | Used for testing on FreeBSD | https://github.com/marketplace/cirrus-ci/ | `.cirrus.yml` | +| Circle CI | Historically was used to provide faster signal,
but we may be able to migrate these to Github Actions | https://circleci.com/docs/2.0/getting-started/#setting-up-circleci
https://youtu.be/Js3hMUsSZ2c
https://circleci.com/docs/2.0/enable-checks/ | `.circleci/config.yml` | + +Note: the instructions linked above mostly cover how to set up a repository with CI from scratch. +The general idea should be the same for setting up CI on your fork of zstd, but you may have to +follow slightly different steps. In particular, please ignore any instructions related to setting up +config files (since zstd already has configs for each of these services). + +## Performance +Performance is extremely important for zstd and we only merge pull requests whose performance +landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented. +This high bar for performance means that every PR which has the potential to +impact performance takes a very long time for us to properly review. That being said, we +always welcome contributions to improve performance (or worsen performance for the trade-off of +something else). Please keep the following in mind before submitting a performance related PR: + +1. Zstd isn't as old as gzip but it has been around for time now and its evolution is +very well documented via past Github issues and pull requests. It may be the case that your +particular performance optimization has already been considered in the past. Please take some +time to search through old issues and pull requests using keywords specific to your +would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected +on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case, +it will be helpful for you to have context from that topic's history before contributing. +2. The distinction between noise and actual performance gains can unfortunately be very subtle +especially when microbenchmarking extremely small wins or losses. The only remedy to getting +something subtle merged is extensive benchmarking. You will be doing us a great favor if you +take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc) +benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark +your changes on every single processor and os out there (and neither will we) but do that best +you can:) We've added some things to think about when benchmarking below in the Benchmarking +Performance section which might be helpful for you. +3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly +legitimate thing to do as long as it does not harm the overall performance health of Zstd. +This is a hard balance to strike but please keep in mind other aspects of Zstd when +submitting changes that are clang-specific, windows-specific, etc. + +## Benchmarking Performance +Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical +testing over theoretical speculation. This guide it not perfect but for most scenarios, it +is a good place to start. + +### Stability +Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable +benchmarking machine. A virtual machine, a machine with shared resources, or your laptop +will typically not be stable enough to obtain reliable benchmark results. If you can get your +hands on a desktop, this is usually a better scenario. + +Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to +do a little more work to ensure that you are in fact measuring the changes you've made not and +noise. Here are some things you can do to make your benchmarks more stable: + +1. The most simple thing you can do to drastically improve the stability of your benchmark is +to run it multiple times and then aggregate the results of those runs. As a general rule of +thumb, the smaller the change you are trying to measure, the more samples of benchmark runs +you will have to aggregate over to get reliable results. Here are some additional things to keep in +mind when running multiple trials: + * How you aggregate your samples are important. You might be tempted to use the mean of your + results. While this is certainly going to be a more stable number than a raw single sample + benchmark number, you might have more luck by taking the median. The mean is not robust to + outliers whereas the median is. Better still, you could simply take the fastest speed your + benchmark achieved on each run since that is likely the fastest your process will be + capable of running your code. In our experience, this (aggregating by just taking the sample + with the fastest running time) has been the most stable approach. + * The more samples you have, the more stable your benchmarks should be. You can verify + your improved stability by looking at the size of your confidence intervals as you + increase your sample count. These should get smaller and smaller. Eventually hopefully + smaller than the performance win you are expecting. + * Most processors will take some time to get `hot` when running anything. The observations + you collect during that time period will very different from the true performance number. Having + a very large number of sample will help alleviate this problem slightly but you can also + address is directly by simply not including the first `n` iterations of your benchmark in + your aggregations. You can determine `n` by simply looking at the results from each iteration + and then hand picking a good threshold after which the variance in results seems to stabilize. +2. You cannot really get reliable benchmarks if your host machine is simultaneously running +another cpu/memory-intensive application in the background. If you are running benchmarks on your +personal laptop for instance, you should close all applications (including your code editor and +browser) before running your benchmarks. You might also have invisible background applications +running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager +on Windows. You will get more stable benchmark results of you end those processes as well. + * If you have multiple cores, you can even run your benchmark on a reserved core to prevent + pollution from other OS and user processes. There are a number of ways to do this depending + on your OS: + * On linux boxes, you have use https://github.com/lpechacek/cpuset. + * On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows + * On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2 +3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd. +Dynamically linking your library will introduce some added variation (not a large amount but +definitely some). Statically linking libzstd will be more stable. Static libraries should +be enabled by default when building zstd. +4. Use a profiler with a good high resolution timer. See the section below on profiling for +details on this. +5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS) +6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on +tmpfs avoids touching a real storage system, which can have a pretty big variability. + +Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html + +### Zstd benchmark +The fastest signal you can get regarding your performance changes is via the in-build zstd cli +bench option. You can run Zstd as you typically would for your scenario using some set of options +and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline +for that options you have just provided. If you want to look at the internals of how this +benchmarking script works, you can check out programs/benchzstd.c + +For example: say you have made a change that you believe improves the speed of zstd level 1. The +very first thing you should use to assess whether you actually achieved any sort of improvement +is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to +specify a running time for your benchmark in seconds (default is 3 seconds). +Usually, the longer the running time, the more stable your results will be. + +``` +$ git checkout +$ make && cp zstd zstd-old +$ git checkout +$ make && cp zstd zstd-new +$ zstd-old -i5 -b1 + 1 : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s +$ zstd-new -i5 -b1 + 1 : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s +``` + +Unless your performance win is large enough to be visible despite the intrinsic noise +on your computer, benchzstd alone will likely not be enough to validate the impact of your +changes. For example, the results of the example above indicate that effectively nothing +changed but there could be a small <3% improvement that the noise on the host machine +obscured. So unless you see a large performance win (10-15% consistently) using just +this method of evaluation will not be sufficient. + +### Profiling +There are a number of great profilers out there. We're going to briefly mention how you can +profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler` +on Windows. + +Say you have an idea for a change that you think will provide some good performance gains +for level 1 compression on Zstd. Typically this means, you have identified a section of +code that you think can be made to run faster. + +The first thing you will want to do is make sure that the piece of code is actually taking up +a notable amount of time to run. It is usually not worth optimizing something which accounts for less than +0.0001% of the total running time. Luckily, there are tools to help with this. +Profilers will let you see how much time your code spends inside a particular function. +If your target code snippet is only part of a function, it might be worth trying to +isolate that snippet by moving it to its own function (this is usually not necessary but +might be). + +Most profilers (including the profilers discussed below) will generate a call graph of +functions for you. Your goal will be to find your function of interest in this call graph +and then inspect the time spent inside of it. You might also want to look at the annotated +assembly which most profilers will provide you with. + +#### Instruments +We will once again consider the scenario where you think you've identified a piece of code +whose performance can be improved upon. Follow these steps to profile your code using +Instruments. + +1. Open Instruments +2. Select `Time Profiler` from the list of standard templates +3. Close all other applications except for your instruments window and your terminal +4. Run your benchmarking script from your terminal window + * You will want a benchmark that runs for at least a few seconds (5 seconds will + usually be long enough). This way the profiler will have something to work with + and you will have ample time to attach your profiler to this process:) + * I will just use benchzstd as my benchmarmking script for this example: +``` +$ zstd -b1 -i5 # this will run for 5 seconds +``` +5. Once you run your benchmarking script, switch back over to instruments and attach your +process to the time profiler. You can do this by: + * Clicking on the `All Processes` drop down in the top left of the toolbar. + * Selecting your process from the dropdown. In my case, it is just going to be labeled + `zstd` + * Hitting the bright red record circle button on the top left of the toolbar +6. You profiler will now start collecting metrics from your benchmarking script. Once +you think you have collected enough samples (usually this is the case after 3 seconds of +recording), stop your profiler. +7. Make sure that in toolbar of the bottom window, `profile` is selected. +8. You should be able to see your call graph. + * If you don't see the call graph or an incomplete call graph, make sure you have compiled + zstd and your benchmarking script using debug flags. On mac and linux, this just means + you will have to supply the `-g` flag alone with your build script. You might also + have to provide the `-fno-omit-frame-pointer` flag +9. Dig down the graph to find your function call and then inspect it by double clicking +the list item. You will be able to see the annotated source code and the assembly side by +side. + +#### Perf + +This wiki has a pretty detailed tutorial on getting started working with perf so we'll +leave you to check that out of you're getting started: + +https://perf.wiki.kernel.org/index.php/Tutorial + +Some general notes on perf: +* Use `perf stat -r # ` to quickly get some relevant timing and +counter statistics. Perf uses a high resolution timer and this is likely one +of the first things your team will run when assessing your PR. +* Perf has a long list of hardware counters that can be viewed with `perf --list`. +When measuring optimizations, something worth trying is to make sure the hardware +counters you expect to be impacted by your change are in fact being so. For example, +if you expect the L1 cache misses to decrease with your change, you can look at the +counter `L1-dcache-load-misses` +* Perf hardware counters will not work on a virtual machine. + +#### Visual Studio + +TODO + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is +clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Coding Style +It's a pretty long topic, which is difficult to summarize in a single paragraph. +As a rule of thumbs, try to imitate the coding style of +similar lines of codes around your contribution. +The following is a non-exhaustive list of rules employed in zstd code base: + +### C90 +This code base is following strict C90 standard, +with 2 extensions : 64-bit `long long` types, and variadic macros. +This rule is applied strictly to code within `lib/` and `programs/`. +Sub-project in `contrib/` are allowed to use other conventions. + +### C++ direct compatibility : symbol mangling +All public symbol declarations must be wrapped in `extern “C” { … }`, +so that this project can be compiled as C++98 code, +and linked into C++ applications. + +### Minimal Frugal +This design requirement is fundamental to preserve the portability of the code base. +#### Dependencies +- Reduce dependencies to the minimum possible level. + Any dependency should be considered “bad” by default, + and only tolerated because it provides a service in a better way than can be achieved locally. + The only external dependencies this repository tolerates are + standard C libraries, and in rare cases, system level headers. +- Within `lib/`, this policy is even more drastic. + The only external dependencies allowed are ``, ``, ``, + and even then, not directly. + In particular, no function shall ever allocate on heap directly, + and must use instead `ZSTD_malloc()` and equivalent. + Other accepted non-symbol headers are `` and ``. +- Within the project, there is a strict hierarchy of dependencies that must be respected. + `programs/` is allowed to depend on `lib/`, but only its public API. + Within `lib/`, `lib/common` doesn't depend on any other directory. + `lib/compress` and `lib/decompress` shall not depend on each other. + `lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`. +#### Resources +- Functions in `lib/` must use very little stack space, + several dozens of bytes max. + Everything larger must use the heap allocator, + or require a scratch buffer to be emplaced manually. + +### Naming +* All public symbols are prefixed with `ZSTD_` + + private symbols, with a scope limited to their own unit, are free of this restriction. + However, since `libzstd` source code can be amalgamated, + each symbol name must attempt to be (and remain) unique. + Avoid too generic names that could become ground for future collisions. + This generally implies usage of some form of prefix. +* For symbols (functions and variables), naming convention is `PREFIX_camelCase`. + + In some advanced cases, one can also find : + - `PREFIX_prefix2_camelCase` + - `PREFIX_camelCase_extendedQualifier` +* Multi-words names generally consist of an action followed by object: + - for example : `ZSTD_createCCtx()` +* Prefer positive actions + - `goBackward` rather than `notGoForward` +* Type names (`struct`, etc.) follow similar convention, + except that they are allowed and even invited to start by an Uppercase letter. + Example : `ZSTD_CCtx`, `ZSTD_CDict` +* Macro names are all Capital letters. + The same composition rules (`PREFIX_NAME_QUALIFIER`) apply. +* File names are all lowercase letters. + The convention is `snake_case`. + File names **must** be unique across the entire code base, + even when they stand in clearly separated directories. + +### Qualifiers +* This code base is `const` friendly, if not `const` fanatical. + Any variable that can be `const` (aka. read-only) **must** be `const`. + Any pointer which content will not be modified must be `const`. + This property is then controlled at compiler level. + `const` variables are an important signal to readers that this variable isn't modified. + Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function. +* If a function must be inlined, mention it explicitly, + using project's own portable macros, such as `FORCE_INLINE_ATTR`, + defined in `lib/common/compiler.h`. + +### Debugging +* **Assertions** are welcome, and should be used very liberally, + to control any condition the code expects for its correct execution. + These assertion checks will be run in debug builds, and disabled in production. +* For traces, this project provides its own debug macros, + in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`. + +### Code documentation +* Avoid code documentation that merely repeats what the code is already stating. + Whenever applicable, prefer employing the code as the primary way to convey explanations. + Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`. + Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`. +* At declaration level, the documentation explains how to use the function or variable + and when applicable why it's needed, of the scenarios where it can be useful. +* At implementation level, the documentation explains the general outline of the algorithm employed, + and when applicable why this specific choice was preferred. + +### General layout +* 4 spaces for indentation rather than tabs +* Code documentation shall directly precede function declaration or implementation +* Function implementations and its code documentation should be preceded and followed by an empty line + + +## License +By contributing to Zstandard, you agree that your contributions will be licensed +under both the [LICENSE](LICENSE) file and the [COPYING](COPYING) file in the root directory of this source tree. diff --git a/deps/zstd/COPYING b/deps/zstd/COPYING new file mode 100644 index 0000000000..ecbc059373 --- /dev/null +++ b/deps/zstd/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/deps/zstd/LICENSE b/deps/zstd/LICENSE new file mode 100644 index 0000000000..75800288cc --- /dev/null +++ b/deps/zstd/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/zstd/Makefile b/deps/zstd/Makefile new file mode 100644 index 0000000000..3b2e3999f4 --- /dev/null +++ b/deps/zstd/Makefile @@ -0,0 +1,443 @@ +# ################################################################ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +# verbose mode (print commands) on V=1 or VERBOSE=1 +Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) + +PRGDIR = programs +ZSTDDIR = lib +BUILDIR = build +ZWRAPDIR = zlibWrapper +TESTDIR = tests +FUZZDIR = $(TESTDIR)/fuzz + +# Define nul output +VOID = /dev/null + +# When cross-compiling from linux to windows, you might +# need to specify this as "Windows." Fedora build fails +# without it. +# +# Note: mingw-w64 build from linux to windows does not +# fail on other tested distros (ubuntu, debian) even +# without manually specifying the TARGET_SYSTEM. +TARGET_SYSTEM ?= $(OS) +CP ?= cp + +ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) + EXT =.exe +else + EXT = +endif + +## default: Build lib-release and zstd-release +.PHONY: default +default: lib-release zstd-release + +.PHONY: all +all: allmost examples manual contrib + +.PHONY: allmost +allmost: allzstd zlibwrapper + +# skip zwrapper, can't build that on alternate architectures without the proper zlib installed +.PHONY: allzstd +allzstd: lib + $(Q)$(MAKE) -C $(PRGDIR) all + $(Q)$(MAKE) -C $(TESTDIR) all + +.PHONY: all32 +all32: + $(MAKE) -C $(PRGDIR) zstd32 + $(MAKE) -C $(TESTDIR) all32 + +.PHONY: lib lib-release lib-mt lib-nomt +lib lib-release lib-mt lib-nomt: + $(Q)$(MAKE) -C $(ZSTDDIR) $@ + +.PHONY: zstd zstd-release +zstd zstd-release: + $(Q)$(MAKE) -C $(PRGDIR) $@ + $(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT) + +.PHONY: zstdmt +zstdmt: + $(Q)$(MAKE) -C $(PRGDIR) $@ + $(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) + +.PHONY: zlibwrapper +zlibwrapper: lib + $(MAKE) -C $(ZWRAPDIR) all + +## test: run long-duration tests +.PHONY: test +DEBUGLEVEL ?= 1 +test: MOREFLAGS += -g -Werror +test: + DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants + $(MAKE) -C $(TESTDIR) $@ + ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@ + +## shortest: same as `make check` +.PHONY: shortest +shortest: + $(Q)$(MAKE) -C $(TESTDIR) $@ + +## check: run basic tests for `zstd` cli +.PHONY: check +check: shortest + +.PHONY: automated_benchmarking +automated_benchmarking: + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: benchmarking +benchmarking: automated_benchmarking + +## examples: build all examples in `examples/` directory +.PHONY: examples +examples: lib + $(MAKE) -C examples all + +## manual: generate API documentation in html format +.PHONY: manual +manual: + $(MAKE) -C contrib/gen_html $@ + +## man: generate man page +.PHONY: man +man: + $(MAKE) -C programs $@ + +## contrib: build all supported projects in `/contrib` directory +.PHONY: contrib +contrib: lib + $(MAKE) -C contrib/pzstd all + $(MAKE) -C contrib/seekable_format/examples all + $(MAKE) -C contrib/seekable_format/tests test + $(MAKE) -C contrib/largeNbDicts all + $(MAKE) -C contrib/externalSequenceProducer all + cd build/single_file_libs/ ; ./build_decoder_test.sh + cd build/single_file_libs/ ; ./build_library_test.sh + +.PHONY: cleanTabs +cleanTabs: + cd contrib; ./cleanTabs + +.PHONY: clean +clean: + $(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID) + $(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) + $(Q)$(MAKE) -C examples/ $@ > $(VOID) + $(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID) + $(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID) + $(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) + $(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID) + $(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) + $(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID) + $(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp* + $(Q)$(RM) -r lz4 + @echo Cleaning completed + +#------------------------------------------------------------------------------ +# make install is validated only for Linux, macOS, Hurd and some BSD targets +#------------------------------------------------------------------------------ +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX)) + +HOST_OS = POSIX + +MKDIR ?= mkdir -p + +HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +EGREP_OPTIONS ?= +ifeq ($(HAVE_COLORNEVER), 1) +EGREP_OPTIONS += --color=never +endif +EGREP = egrep $(EGREP_OPTIONS) + +# Print a two column output of targets and their description. To add a target description, put a +# comment in the Makefile with the format "## : ". For example: +# +## list: Print all targets and their descriptions (if provided) +.PHONY: list +list: + $(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ + | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ + | $(EGREP) -v -e '^[^[:alnum:]]' | sort); \ + { \ + printf "Target Name\tDescription\n"; \ + printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \ + for target in $$TARGETS; do \ + line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \ + description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \ + printf "$$target\t$$description\n"; \ + done \ + } | column -t -s $$'\t' + +.PHONY: install armtest usan asan uasan msan asan32 +install: + $(Q)$(MAKE) -C $(ZSTDDIR) $@ + $(Q)$(MAKE) -C $(PRGDIR) $@ + +.PHONY: uninstall +uninstall: + $(Q)$(MAKE) -C $(ZSTDDIR) $@ + $(Q)$(MAKE) -C $(PRGDIR) $@ + +.PHONY: travis-install +travis-install: + $(MAKE) install PREFIX=~/install_test_dir + +.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build +gcc5build: clean + gcc-5 -v + CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" + +gcc6build: clean + gcc-6 -v + CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" + +gcc7build: clean + gcc-7 -v + CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" + +clangbuild: clean + clang -v + CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all + +m32build: clean + gcc -v + $(MAKE) all32 + +armbuild: clean + CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd + +aarch64build: clean + CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd + +ppcbuild: clean + CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd + +ppc64build: clean + CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd + +.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz +armfuzz: clean + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest + +aarch64fuzz: clean + ld -v + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest + +ppcfuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest + +ppc64fuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest + +.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test +cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror +cxxtest: clean + $(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)" # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly + +gcc5test: clean + gcc-5 -v + $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)" + +gcc6test: clean + gcc-6 -v + $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)" + +armtest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" + +aarch64test: + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" + +ppctest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" + +ppc64test: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" + +.PHONY: arm-ppc-compilation +arm-ppc-compilation: + $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" + $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" + +regressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest + +uasanregressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined" + +msanregressiontest: + $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory" + +update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/ +update_regressionResults: + $(MAKE) -C programs zstd + $(MAKE) -C tests/regression test + $(RM) -rf $(REGRESS_RESULTS_DIR) + $(MKDIR) $(REGRESS_RESULTS_DIR) + ./tests/regression/test \ + --cache tests/regression/cache \ + --output $(REGRESS_RESULTS_DIR)/results.csv \ + --zstd programs/zstd + echo "Showing results differences" + ! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv + echo "Updating results.csv" + $(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv + + +# run UBsan with -fsanitize-recover=pointer-overflow +# this only works with recent compilers such as gcc 8+ +usan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror $(MOREFLAGS)" + +asan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)" + +asan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* + +msan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason + +msan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* + +asan32: clean + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)" + +uasan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" + +uasan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* + +tsan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" + +.PHONY: apt-install +apt-install: + # TODO: uncomment once issue 3011 is resolved and remove hack from Github Actions .yml + # sudo apt-get update + sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) + +.PHONY: apt-add-repo +apt-add-repo: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + +.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install +ppcinstall: + APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install + +arminstall: + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + +valgrindinstall: + APT_PACKAGES="valgrind" $(MAKE) apt-install + +libc6install: + APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install + +gcc6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gcc7install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install + +gcc8install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install + +gpp6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install + +clang38install: + APT_PACKAGES="clang-3.8" $(MAKE) apt-install + +# Ubuntu 14.04 ships a too-old lz4 +lz4install: + [ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install + +endif + + +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON +endif + +#------------------------------------------------------------------------ +# target specific tests +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze +cmakebuild: + cmake --version + $(RM) -r $(BUILDIR)/cmake/build + $(MKDIR) $(BUILDIR)/cmake/build + cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. + $(MAKE) -C $(BUILDIR)/cmake/build -j4; + $(MAKE) -C $(BUILDIR)/cmake/build install; + $(MAKE) -C $(BUILDIR)/cmake/build uninstall; + cd $(BUILDIR)/cmake/build; ctest -V -L Medium + +c89build: clean + $(CC) -v + CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd + +gnu90build: clean + $(CC) -v + CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost + +c99build: clean + $(CC) -v + CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost + +gnu99build: clean + $(CC) -v + CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost + +c11build: clean + $(CC) -v + CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost + +bmix64build: clean + $(CC) -v + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test + +bmix32build: clean + $(CC) -v + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test + +bmi32build: clean + $(CC) -v + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test + +# static analyzer test uses clang's scan-build +# does not analyze zlibWrapper, due to detected issues in zlib source code +staticAnalyze: SCANBUILD ?= scan-build +staticAnalyze: + $(CC) -v + CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd +endif diff --git a/deps/zstd/Package.swift b/deps/zstd/Package.swift new file mode 100644 index 0000000000..97f2c6a525 --- /dev/null +++ b/deps/zstd/Package.swift @@ -0,0 +1,36 @@ +// swift-tools-version:5.0 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "zstd", + platforms: [ + .macOS(.v10_10), .iOS(.v9), .tvOS(.v9) + ], + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "libzstd", + targets: [ "libzstd" ]) + ], + dependencies: [ + // Dependencies declare other packages that this package depends on. + // .package(url: /* package url */, from: "1.0.0"), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "libzstd", + path: "lib", + sources: [ "common", "compress", "decompress", "dictBuilder" ], + publicHeadersPath: ".", + cSettings: [ + .headerSearchPath(".") + ]) + ], + swiftLanguageVersions: [.v5], + cLanguageStandard: .gnu11, + cxxLanguageStandard: .gnucxx14 +) diff --git a/deps/zstd/README.md b/deps/zstd/README.md new file mode 100644 index 0000000000..f91e68fdb1 --- /dev/null +++ b/deps/zstd/README.md @@ -0,0 +1,223 @@ +

Zstandard

+ +__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, +targeting real-time compression scenarios at zlib-level and better compression ratios. +It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy). + +Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available. +This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library, +and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files. +Should your project require another programming language, +a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages). + +**Development branch status:** + +[![Build Status][travisDevBadge]][travisLink] +[![Build status][CircleDevBadge]][CircleLink] +[![Build status][CirrusDevBadge]][CirrusLink] +[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink] + +[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite" +[travisLink]: https://travis-ci.com/facebook/zstd +[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" +[CircleLink]: https://circleci.com/gh/facebook/zstd +[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev +[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd +[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg +[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd + +## Benchmarks + +For reference, several fast compression algorithms were tested and compared +on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`), +with a Core i7-9700K CPU @ 4.9GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 9.3.0, +on the [Silesia compression corpus]. + +[lzbench]: https://github.com/inikep/lzbench +[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia +[gcc]: https://gcc.gnu.org/ + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.5.1 -1** | 2.887 | 530 MB/s | 1700 MB/s | +| [zlib] 1.2.11 -1 | 2.743 | 95 MB/s | 400 MB/s | +| brotli 1.0.9 -0 | 2.702 | 395 MB/s | 450 MB/s | +| **zstd 1.5.1 --fast=1** | 2.437 | 600 MB/s | 2150 MB/s | +| **zstd 1.5.1 --fast=3** | 2.239 | 670 MB/s | 2250 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 540 MB/s | 760 MB/s | +| **zstd 1.5.1 --fast=4** | 2.148 | 710 MB/s | 2300 MB/s | +| lzo1x 2.10 -1 | 2.106 | 660 MB/s | 845 MB/s | +| [lz4] 1.9.3 | 2.101 | 740 MB/s | 4500 MB/s | +| lzf 3.6 -1 | 2.077 | 410 MB/s | 830 MB/s | +| snappy 1.1.9 | 2.073 | 550 MB/s | 1750 MB/s | + +[zlib]: https://www.zlib.net/ +[lz4]: https://lz4.github.io/lz4/ + +The negative compression levels, specified with `--fast=#`, +offer faster compression and decompression speed +at the cost of compression ratio (compared to level 1). + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. +Decompression speed is preserved and remains roughly the same at all settings, +a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 7.3.0, +on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- +![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed") + +A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph. +For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png). + + +## The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighing about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- +![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed") + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +### Dictionary compression How To: + +1. Create the dictionary + + `zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2. Compress with dictionary + + `zstd -D dictionaryName FILE` + +3. Decompress with dictionary + + `zstd -D dictionaryName --decompress FILE.zst` + + +## Build instructions + +`make` is the officially maintained build system of this project. +All other build systems are "compatible" and 3rd-party maintained, +they may feature small differences in advanced options. +When your system allows it, prefer using `make` to build `zstd` and `libzstd`. + +### Makefile + +If your system is compatible with standard `make` (or `gmake`), +invoking `make` in root directory will generate `zstd` cli in root directory. +It will also create `libzstd` into `lib/`. + +Other available options include: +- `make install` : create and install zstd cli, library and man pages +- `make check` : create and run `zstd`, test its behavior on local platform + +The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), +allowing staged install, standard flags, directory variables and command variables. + +For advanced use cases, specialized compilation flags which control binary generation +are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library +and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI. + +### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + +#### Support for Fat (Universal2) Output + +`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support. +To perform a Fat/Universal2 build and install use the following commands: + +```bash +cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64" +cd build-cmake-debug +ninja +sudo ninja install +``` + +### Meson + +A Meson project is provided within [`build/meson`](build/meson). Follow +build instructions in that directory. + +You can also take a look at [`.travis.yml`](.travis.yml) file for an +example about how Meson is used to build this project. + +Note that default build type is **release**. + +### VCPKG +You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install zstd + +The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities: +- Projects for Visual Studio 2005, 2008 and 2010. + + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017. +- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + +### Buck + +You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. +The output binary will be in `buck-out/gen/programs/`. + +## Testing + +You can run quick local smoke tests by running `make check`. +If you can't use `make`, execute the `playTest.sh` script from the `src/tests` directory. +Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the `zstd` and `datagen` binary. +For information on CI testing, please refer to `TESTING.md`. + +## Status + +Zstandard is currently deployed within Facebook and many other large cloud infrastructures. +It is run continuously to compress large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +## License + +Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). + +## Contributing + +The `dev` branch is the one where all contributions are merged before reaching `release`. +If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch. +Direct commit to `release` are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). diff --git a/deps/zstd/TESTING.md b/deps/zstd/TESTING.md new file mode 100644 index 0000000000..df842cc941 --- /dev/null +++ b/deps/zstd/TESTING.md @@ -0,0 +1,43 @@ +Testing +======= + +Zstandard CI testing is split up into three sections: +short, medium, and long tests. + +Short Tests +----------- +Short tests run on CircleCI for new commits on every branch and pull request. +They consist of the following tests: +- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64) +- Compilation on various versions of gcc, clang, and g++ +- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests) +- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64 + +Medium Tests +------------ +Medium tests run on every commit and pull request to `dev` branch, on TravisCI. +They consist of the following tests: +- The following tests run with UBsan and Asan on x86_64 and x86, as well as with + Msan on x86_64 + - `tests/playTests.sh --test-large-data` + - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c` +- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode) +- Valgrind Test (`make -C tests test-valgrind`) (testing CLI and fuzzer under `valgrind`) +- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64 + +Long Tests +---------- +Long tests run on all commits to `release` branch, +and once a day on the current version of `dev` branch, +on TravisCI. +They consist of the following tests: +- Entire test suite (including fuzzers and some other specialized tests) on: + - x86_64 and x86 with UBsan and Asan + - x86_64 with Msan + - ARM, AArch64, PowerPC, and PowerPC64 +- Streaming mode fuzzer with Tsan (for the `zstdmt` testing) +- ZlibWrapper tests, including under valgrind +- Versions test (ensuring `zstd` can decode files from all previous versions) +- `pzstd` with asan and tsan, as well as in 32-bits mode +- Testing `zstd` with legacy mode off +- Entire test suite and make install on macOS diff --git a/deps/zstd/lib/README.md b/deps/zstd/lib/README.md new file mode 100644 index 0000000000..c3b5d1817f --- /dev/null +++ b/deps/zstd/lib/README.md @@ -0,0 +1,224 @@ +Zstandard library files +================================ + +The __lib__ directory is split into several sub-directories, +in order to make it easier to select or exclude features. + + +#### Building + +`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), +including commands variables, staged install, directory variables and standard targets. +- `make` : generates both static and dynamic libraries +- `make install` : install libraries and headers in target system directories + +`libzstd` default scope is pretty large, including compression, decompression, dictionary builder, +and support for decoding legacy formats >= v0.5.0. +The scope can be reduced on demand (see paragraph _modular build_). + + +#### Multithreading support + +When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons). + +Enabling multithreading requires 2 conditions : +- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) +- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + +For convenience, we provide a build target to generate multi and single threaded libraries: +- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`. +- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`. +- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`. + +When linking a POSIX program with a multithreaded version of `libzstd`, +note that it's necessary to invoke the `-pthread` flag during link stage. + +Multithreading capabilities are exposed +via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351). + + +#### API + +Zstandard's stable API is exposed within [lib/zstd.h](zstd.h). + + +#### Advanced API + +Optional advanced features are exposed via : + +- `lib/zstd_errors.h` : translates `size_t` function results + into a `ZSTD_ErrorCode`, for accurate error handling. + +- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`, + it unlocks access to the experimental API, + exposed in the second part of `zstd.h`. + All definitions in the experimental APIs are unstable, + they may still change in the future, or even be removed. + As a consequence, experimental definitions shall ___never be used with dynamic library___ ! + Only static linking is allowed. + + +#### Modular build + +It's possible to compile only a limited set of features within `libzstd`. +The file structure is designed to make this selection manually achievable for any build system : + +- Directory `lib/common` is always required, for all variants. + +- Compression source code lies in `lib/compress` + +- Decompression source code lies in `lib/decompress` + +- It's possible to include only `compress` or only `decompress`, they don't depend on each other. + +- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples. + The API is exposed in `lib/dictBuilder/zdict.h`. + This module depends on both `lib/common` and `lib/compress` . + +- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`. + This module depends on `lib/common` and `lib/decompress`. + To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation. + Specifying a number limits versions supported to that version onward. + For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0". + Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats". + By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`. + Decoding supported legacy format is a transparent capability triggered within decompression functions. + It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`. + Each version does also provide its own set of advanced API. + For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` . + +- While invoking `make libzstd`, it's possible to define build macros + `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, + and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the + corresponding features. This will also disable compilation of all + dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable + dictBuilder). + +- There are a number of options that can help minimize the binary size of + `libzstd`. + + The first step is to select the components needed (using the above-described + `ZSTD_LIB_COMPRESSION` etc.). + + The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This + disables various optional components and changes the compilation flags to + prioritize space-saving. + + Detailed options: Zstandard's code and build environment is set up by default + to optimize above all else for performance. In pursuit of this goal, Zstandard + makes significant trade-offs in code size. For example, Zstandard often has + more than one implementation of a particular component, with each + implementation optimized for different scenarios. For example, the Huffman + decoder has complementary implementations that decode the stream one symbol at + a time or two symbols at a time. Zstd normally includes both (and dispatches + between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or + `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding + compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` + and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of + only one or the other of two decompression implementations. The smallest + binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and + `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`). + + For squeezing the last ounce of size out, you can also define + `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`, + which removes the error messages that are otherwise returned by + `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`). + + Finally, when integrating into your application, make sure you're doing link- + time optimization and unused symbol garbage collection (via some combination of, + e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`, + `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`, + `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands + the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your + compiler's documentation. + +- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1` + will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in + the shared library, which is now hidden by default. + +- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries + which can detect at runtime the presence of BMI2 instructions, and use them only if present. + These instructions contribute to better performance, notably on the decoder side. + By default, this feature is automatically enabled on detecting + the right instruction set (x64) and compiler (clang or gcc >= 5). + It's obviously disabled for different cpus, + or when BMI2 instruction set is _required_ by the compiler command line + (in this case, only the BMI2 code path is generated). + Setting this macro will either force to generate the BMI2 dispatcher (1) + or prevent it (0). It overrides automatic detection. + +- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions + that zstd does not use. Not all unused functions are hidden, but they can be if needed. + Currently, this macro will hide function definitions in FSE and HUF that use an excessive + amount of stack space. + +- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. + Compiler builtins are still used. + +- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control + the amount of extra memory used during decompression to store literals. + This defaults to 64kB. Reducing this value reduces the memory footprint of + `ZSTD_DCtx` decompression contexts, + but might also result in a small decompression speed cost. + +- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE` + can be overridden to control the visibility of zstd's API. Additionally, + `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility + of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols + from the shared library. These macros default to `ZSTDLIB_VISIBILITY`, + `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility + with the old macro names. + +#### Windows : using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make libzstd` command. +This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`. +The import library is only required with Visual C++. +The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to +compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses ZSTD consists of a single `test-dll.c` +file it should be linked with `dll\libzstd.dll`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll +``` +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. + + +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + +#### Deprecated API + +Obsolete API on their way out are stored in directory `lib/deprecated`. +At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`. +These prototypes will be removed in some future version. +Consider migrating code towards supported streaming API exposed in `zstd.h`. + + +#### Miscellaneous + +The other files are not source code. There are : + + - `BUCK` : support for `buck` build system (https://buckbuild.com/) + - `Makefile` : `make` script to build and install zstd library (static and dynamic) + - `README.md` : this file + - `dll/` : resources directory for Windows compilation + - `libzstd.pc.in` : script for `pkg-config` (used in `make install`) diff --git a/deps/zstd/lib/common/allocations.h b/deps/zstd/lib/common/allocations.h new file mode 100644 index 0000000000..a3153c4bac --- /dev/null +++ b/deps/zstd/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "mem.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/deps/zstd/lib/common/bits.h b/deps/zstd/lib/common/bits.h new file mode 100644 index 0000000000..def56c474c --- /dev/null +++ b/deps/zstd/lib/common/bits.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/deps/zstd/lib/common/bitstream.h b/deps/zstd/lib/common/bitstream.h new file mode 100644 index 0000000000..72b0b3df22 --- /dev/null +++ b/deps/zstd/lib/common/bitstream.h @@ -0,0 +1,437 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/deps/zstd/lib/common/compiler.h b/deps/zstd/lib/common/compiler.h new file mode 100644 index 0000000000..73f8d01998 --- /dev/null +++ b/deps/zstd/lib/common/compiler.h @@ -0,0 +1,358 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); } +#else +# define ZSTD_UNREACHABLE { assert(0); } +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/deps/zstd/lib/common/cpu.h b/deps/zstd/lib/common/cpu.h new file mode 100644 index 0000000000..8bc34a36da --- /dev/null +++ b/deps/zstd/lib/common/cpu.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/deps/zstd/lib/common/debug.c b/deps/zstd/lib/common/debug.c new file mode 100644 index 0000000000..ebf7bfccfa --- /dev/null +++ b/deps/zstd/lib/common/debug.c @@ -0,0 +1,24 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/deps/zstd/lib/common/debug.h b/deps/zstd/lib/common/debug.h new file mode 100644 index 0000000000..0e9817ea6d --- /dev/null +++ b/deps/zstd/lib/common/debug.h @@ -0,0 +1,107 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } } +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } } +#else +# define RAWLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/deps/zstd/lib/common/entropy_common.c b/deps/zstd/lib/common/entropy_common.c new file mode 100644 index 0000000000..e2173afb0a --- /dev/null +++ b/deps/zstd/lib/common/entropy_common.c @@ -0,0 +1,340 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = ZSTD_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int flags) +{ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)flags; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/deps/zstd/lib/common/error_private.c b/deps/zstd/lib/common/error_private.c new file mode 100644 index 0000000000..075fc5ef42 --- /dev/null +++ b/deps/zstd/lib/common/error_private.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Data corruption detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/deps/zstd/lib/common/error_private.h b/deps/zstd/lib/common/error_private.h new file mode 100644 index 0000000000..325daad404 --- /dev/null +++ b/deps/zstd/lib/common/error_private.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/deps/zstd/lib/common/fse.h b/deps/zstd/lib/common/fse.h new file mode 100644 index 0000000000..02a1f0bc53 --- /dev/null +++ b/deps/zstd/lib/common/fse.h @@ -0,0 +1,639 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/deps/zstd/lib/common/fse_decompress.c b/deps/zstd/lib/common/fse_decompress.c new file mode 100644 index 0000000000..1e1c9f92d6 --- /dev/null +++ b/deps/zstd/lib/common/fse_decompress.c @@ -0,0 +1,311 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" +#include "bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/deps/zstd/lib/common/huf.h b/deps/zstd/lib/common/huf.h new file mode 100644 index 0000000000..73d1ee5654 --- /dev/null +++ b/deps/zstd/lib/common/huf.h @@ -0,0 +1,273 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ + +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; + + +/* **************************************** + * HUF detailed API + * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; + +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int flags); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#endif +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif + +#endif /* HUF_H_298734234 */ + +#if defined (__cplusplus) +} +#endif diff --git a/deps/zstd/lib/common/mem.h b/deps/zstd/lib/common/mem.h new file mode 100644 index 0000000000..98dd47a047 --- /dev/null +++ b/deps/zstd/lib/common/mem.h @@ -0,0 +1,435 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * Default : method 1 if supported, else method 0 + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# ifdef __GNUC__ +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return MEM_swap32_fallback(in); +#endif +} + +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/deps/zstd/lib/common/pool.c b/deps/zstd/lib/common/pool.c new file mode 100644 index 0000000000..d5ca5a7808 --- /dev/null +++ b/deps/zstd/lib/common/pool.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/deps/zstd/lib/common/pool.h b/deps/zstd/lib/common/pool.h new file mode 100644 index 0000000000..eb22ff509f --- /dev/null +++ b/deps/zstd/lib/common/pool.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and re-use existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/deps/zstd/lib/common/portability_macros.h b/deps/zstd/lib/common/portability_macros.h new file mode 100644 index 0000000000..8fd6ea82d1 --- /dev/null +++ b/deps/zstd/lib/common/portability_macros.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/deps/zstd/lib/common/threading.c b/deps/zstd/lib/common/threading.c new file mode 100644 index 0000000000..ca155b9b9d --- /dev/null +++ b/deps/zstd/lib/common/threading.c @@ -0,0 +1,176 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + +static unsigned __stdcall worker(void *arg) +{ + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + ZSTD_thread_params_t thread_param; + (void)unused; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + *thread = NULL; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (!thread) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return errno; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread) +{ + DWORD result; + + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + + switch (result) { + case WAIT_OBJECT_0: + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/deps/zstd/lib/common/threading.h b/deps/zstd/lib/common/threading.h new file mode 100644 index 0000000000..fb5c1c8787 --- /dev/null +++ b/deps/zstd/lib/common/threading.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef HANDLE ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/deps/zstd/lib/common/xxhash.c b/deps/zstd/lib/common/xxhash.c new file mode 100644 index 0000000000..fd237c9062 --- /dev/null +++ b/deps/zstd/lib/common/xxhash.c @@ -0,0 +1,24 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/deps/zstd/lib/common/xxhash.h b/deps/zstd/lib/common/xxhash.h new file mode 100644 index 0000000000..b8b73290bb --- /dev/null +++ b/deps/zstd/lib/common/xxhash.h @@ -0,0 +1,5686 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (e.g. eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputv[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || defined(__aarch64__) || defined(_M_ARM) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__arm__) || defined(__thumb__) || defined(_M_ARM)) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and + * 2 lanes on scalar by default. + * + * This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the + * emulated 64-bit arithmetic is too slow. + * + * Modern ARM CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't + * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. + * + * This is even more noticeable on the more advanced cores like the A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the + * remaining lanes will use scalar instructions. This improves the bandwidth + * and also gives the integer pipelines something to do besides twiddling loop + * counters and pointers. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__OPTIMIZE_SIZE__) +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned int* const xacc = (unsigned int*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripesToEnd <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/deps/zstd/lib/common/zstd_internal.h b/deps/zstd/lib/common/zstd_internal.h new file mode 100644 index 0000000000..1f942f27bf --- /dev/null +++ b/deps/zstd/lib/common/zstd_internal.h @@ -0,0 +1,392 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/deps/zstd/lib/common/zstd_trace.h b/deps/zstd/lib/common/zstd_trace.h new file mode 100644 index 0000000000..da20534ebd --- /dev/null +++ b/deps/zstd/lib/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386 and aarch64 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/deps/zstd/lib/decompress/huf_decompress.c b/deps/zstd/lib/decompress/huf_decompress.c new file mode 100644 index 0000000000..5b217ac586 --- /dev/null +++ b/deps/zstd/lib/decompress/huf_decompress.c @@ -0,0 +1,1882 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilimit [in] - The input limit, stop when any input pointer is below ilimit. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilimit, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilimit; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + + BYTE* const oend = (BYTE*)dst + dstSize; + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + const BYTE* const istart = (const BYTE*)src; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + * length1 must be >= 16 so that ip[0] >= ilimit before the loop + * starts. + */ + if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* If ip[] >= ilimit, it is guaranteed to be safe to + * reload bits[]. It may be beyond its section, but is + * guaranteed to be valid (>= istart). + */ + args->ilimit = ilimit; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->iend[0]; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilimit) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we get close to the end. */ + if (op[3] + 20 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Decode 5 symbols in each of the 4 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 4; ++stream) { + int const index = (int)(bits[stream] >> 53); + int const entry = (int)dtable[index]; + bits[stream] <<= (entry & 63); + op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF); + } + } + /* Reload the bitstreams */ + for (stream = 0; stream < 4; ++stream) { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + op[stream] += 5; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilimit and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilimit) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop if we are too close to the end. */ + if (op[3] + 10 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Do 5 table lookups for each of the first 3 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 3; ++stream) { + int const index = (int)(bits[stream] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[stream], entry.sequence); + bits[stream] <<= (entry.nbBits); + op[stream] += (entry.length); + } + } + /* Do 1 table lookup from the final stream */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Do 4 table lookups from the final stream & reload bitstreams */ + for (stream = 0; stream < 4; ++stream) { + /* Do a table lookup from the final stream. + * This is interleaved with the reloading to reduce register + * pressure. This shouldn't be necessary, but compilers can + * struggle with codegen with high register pressure. + */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Reload the bistreams. The final bitstream must be reloaded + * after the 5th symbol was decoded. + */ + { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} diff --git a/deps/zstd/lib/decompress/huf_decompress_amd64.S b/deps/zstd/lib/decompress/huf_decompress_amd64.S new file mode 100644 index 0000000000..671624fe34 --- /dev/null +++ b/deps/zstd/lib/decompress/huf_decompress_amd64.S @@ -0,0 +1,576 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/portability_macros.h" + +/* Stack marking + * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart + */ +#if defined(__ELF__) && defined(__GNUC__) +.section .note.GNU-stack,"",%progbits +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +/* Calling convention: + * + * %rdi contains the first argument: HUF_DecompressAsmArgs*. + * %rbp isn't maintained (no frame pointer). + * %rsp contains the stack pointer that grows down. + * No red-zone is assumed, only addresses >= %rsp are used. + * All register contents are preserved. + * + * TODO: Support Windows calling convention. + */ + +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.text + +/* Sets up register mappings for clarity. + * op[], bits[], dtable & ip[0] each get their own register. + * ip[1,2,3] & olimit alias var[]. + * %rax is a scratch register. + */ + +#define op0 rsi +#define op1 rbx +#define op2 rcx +#define op3 rdi + +#define ip0 r8 +#define ip1 r9 +#define ip2 r10 +#define ip3 r11 + +#define bits0 rbp +#define bits1 rdx +#define bits2 r12 +#define bits3 r13 +#define dtable r14 +#define olimit r15 + +/* var[] aliases ip[1,2,3] & olimit + * ip[1,2,3] are saved every iteration. + * olimit is only used in compute_olimit. + */ +#define var0 r15 +#define var1 r9 +#define var2 r10 +#define var3 r11 + +/* 32-bit var registers */ +#define vard0 r15d +#define vard1 r9d +#define vard2 r10d +#define vard3 r11d + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM(X) \ + X(0); \ + X(1); \ + X(2); \ + X(3) + +/* Calls X(N, idx) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \ + X(0, idx); \ + X(1, idx); \ + X(2, idx); \ + X(3, idx) + +/* Define both _HUF_* & HUF_* symbols because MacOS + * C symbols are prefixed with '_' & Linux symbols aren't. + */ +_HUF_decompress4X1_usingDTable_internal_fast_asm_loop: +HUF_decompress4X1_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + /* Read HUF_DecompressAsmArgs* args from %rax */ + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push 104(%rax) /* ilimit */ + push 112(%rax) /* oend */ + push %olimit /* olimit space */ + + subq $24, %rsp + +.L_4X1_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rbx, rdx must be saved + * op3 & ip0 mustn't be clobbered + */ + movq %rbx, 0(%rsp) + movq %rdx, 8(%rsp) + + movq 32(%rsp), %rax /* rax = oend */ + subq %op3, %rax /* rax = oend - op3 */ + + /* r15 = (oend - op3) / 5 */ + movabsq $-3689348814741910323, %rdx + mulq %rdx + movq %rdx, %r15 + shrq $2, %r15 + + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %rbx /* rbx = ip0 - ilimit */ + + /* rdx = (ip0 - ilimit) / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %rbx + shrq %rbx + addq %rbx, %rdx + shrq $2, %rdx + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* r15 = r15 * 5 */ + leaq (%r15, %r15, 4), %r15 + + /* olimit = op3 + r15 */ + addq %op3, %olimit + + movq 8(%rsp), %rdx + movq 0(%rsp), %rbx + + /* If (op3 + 20 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $20, %rax /* rax = op3 + 20 */ + cmpq %rax, %olimit /* op3 + 20 > olimit */ + jb .L_4X1_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X1_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X1_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X1_exit + +/* Reads top 11 bits from bits[n] + * Loads dt[bits[n]] into var[n] + */ +#define GET_NEXT_DELT(n) \ + movq $53, %var##n; \ + shrxq %var##n, %bits##n, %var##n; \ + movzwl (%dtable,%var##n,2),%vard##n + +/* var[n] must contain the DTable entry computed with GET_NEXT_DELT + * Moves var[n] to %rax + * bits[n] <<= var[n] & 63 + * op[n][idx] = %rax >> 8 + * %ah is a way to access bits [8, 16) of %rax + */ +#define DECODE_FROM_DELT(n, idx) \ + movq %var##n, %rax; \ + shlxq %var##n, %bits##n, %bits##n; \ + movb %ah, idx(%op##n) + +/* Assumes GET_NEXT_DELT has been called. + * Calls DECODE_FROM_DELT then GET_NEXT_DELT + */ +#define DECODE_AND_GET_NEXT(n, idx) \ + DECODE_FROM_DELT(n, idx); \ + GET_NEXT_DELT(n) \ + +/* // ctz & nbBytes is stored in bits[n] + * // nbBits is stored in %rax + * ctz = CTZ[bits[n]] + * nbBits = ctz & 7 + * nbBytes = ctz >> 3 + * op[n] += 5 + * ip[n] -= nbBytes + * // Note: x86-64 is little-endian ==> no bswap + * bits[n] = MEM_readST(ip[n]) | 1 + * bits[n] <<= nbBits + */ +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + andq $7, %rax; \ + shrq $3, %bits##n; \ + leaq 5(%op##n), %op##n; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlx %rax, %bits##n, %bits##n + + /* Store clobbered variables on the stack */ + movq %olimit, 24(%rsp) + movq %ip1, 0(%rsp) + movq %ip2, 8(%rsp) + movq %ip3, 16(%rsp) + + /* Call GET_NEXT_DELT for each stream */ + FOR_EACH_STREAM(GET_NEXT_DELT) + + .p2align 6 + +.L_4X1_loop_body: + /* Decode 5 symbols in each of the 4 streams (20 total) + * Must have called GET_NEXT_DELT for each stream + */ + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4) + + /* Load ip[1,2,3] from stack (var[] aliases them) + * ip[] is needed for RELOAD_BITS + * Each will be stored back to the stack after RELOAD + */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Reload each stream & fetch the next table entry + * to prepare for the next iteration + */ + RELOAD_BITS(0) + GET_NEXT_DELT(0) + + RELOAD_BITS(1) + movq %ip1, 0(%rsp) + GET_NEXT_DELT(1) + + RELOAD_BITS(2) + movq %ip2, 8(%rsp) + GET_NEXT_DELT(2) + + RELOAD_BITS(3) + movq %ip3, 16(%rsp) + GET_NEXT_DELT(3) + + /* If op3 < olimit: continue the loop */ + cmp %op3, 24(%rsp) + ja .L_4X1_loop_body + + /* Reload ip[1,2,3] from stack */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Re-compute olimit */ + jmp .L_4X1_compute_olimit + +#undef GET_NEXT_DELT +#undef DECODE_FROM_DELT +#undef DECODE +#undef RELOAD_BITS +.L_4X1_exit: + addq $24, %rsp + + /* Restore stack (oend & olimit) */ + pop %rax /* olimit */ + pop %rax /* oend */ + pop %rax /* ilimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +_HUF_decompress4X2_usingDTable_internal_fast_asm_loop: +HUF_decompress4X2_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push %rax /* olimit */ + push 104(%rax) /* ilimit */ + + movq 112(%rax), %rax + push %rax /* oend3 */ + + movq %op3, %rax + push %rax /* oend2 */ + + movq %op2, %rax + push %rax /* oend1 */ + + movq %op1, %rax + push %rax /* oend0 */ + + /* Scratch space */ + subq $8, %rsp + +.L_4X2_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rdx must be saved + * op[1,2,3,4] & ip0 mustn't be clobbered + */ + movq %rdx, 0(%rsp) + + /* We can consume up to 7 input bytes each iteration. */ + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %r15 /* r15 = ip0 - ilimit */ + + /* rdx = rax / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %r15 + shrq %r15 + addq %r15, %rdx + shrq $2, %rdx + + /* r15 = (ip0 - ilimit) / 7 */ + movq %rdx, %r15 + + /* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */ + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + movq 16(%rsp), %rdx /* rdx = oend1 */ + subq %op1, %rdx /* rdx = oend1 - op1 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 24(%rsp), %rax /* rax = oend2 */ + subq %op2, %rax /* rax = oend2 - op2 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 32(%rsp), %rax /* rax = oend3 */ + subq %op3, %rax /* rax = oend3 - op3 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movabsq $-3689348814741910323, %rax + mulq %rdx + shrq $3, %rdx /* rdx = rdx / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* olimit = op3 + 5 * r15 */ + movq %r15, %rax + leaq (%op3, %rax, 4), %olimit + addq %rax, %olimit + + movq 0(%rsp), %rdx + + /* If (op3 + 10 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $10, %rax /* rax = op3 + 10 */ + cmpq %rax, %olimit /* op3 + 10 > olimit */ + jb .L_4X2_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X2_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X2_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X2_exit + +#define DECODE(n, idx) \ + movq %bits##n, %rax; \ + shrq $53, %rax; \ + movzwl 0(%dtable,%rax,4),%r8d; \ + movzbl 2(%dtable,%rax,4),%r15d; \ + movzbl 3(%dtable,%rax,4),%eax; \ + movw %r8w, (%op##n); \ + shlxq %r15, %bits##n, %bits##n; \ + addq %rax, %op##n + +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + shrq $3, %bits##n; \ + andq $7, %rax; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlxq %rax, %bits##n, %bits##n + + + movq %olimit, 48(%rsp) + + .p2align 6 + +.L_4X2_loop_body: + /* We clobber r8, so store it on the stack */ + movq %r8, 0(%rsp) + + /* Decode 5 symbols from each of the 4 streams (20 symbols total). */ + FOR_EACH_STREAM_WITH_INDEX(DECODE, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 4) + + /* Reload r8 */ + movq 0(%rsp), %r8 + + FOR_EACH_STREAM(RELOAD_BITS) + + cmp %op3, 48(%rsp) + ja .L_4X2_loop_body + jmp .L_4X2_compute_olimit + +#undef DECODE +#undef RELOAD_BITS +.L_4X2_exit: + addq $8, %rsp + /* Restore stack (oend & olimit) */ + pop %rax /* oend0 */ + pop %rax /* oend1 */ + pop %rax /* oend2 */ + pop %rax /* oend3 */ + pop %rax /* ilimit */ + pop %rax /* olimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +#endif diff --git a/deps/zstd/lib/decompress/zstd_ddict.c b/deps/zstd/lib/decompress/zstd_ddict.c new file mode 100644 index 0000000000..309ec0d036 --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/deps/zstd/lib/decompress/zstd_ddict.h b/deps/zstd/lib/decompress/zstd_ddict.h new file mode 100644 index 0000000000..c4ca8877a0 --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/deps/zstd/lib/decompress/zstd_decompress.c b/deps/zstd/lib/decompress/zstd_decompress.c new file mode 100644 index 0000000000..7bc2713429 --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_decompress.c @@ -0,0 +1,2355 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memmove(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + BYTE* oBlockEnd = oend; + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming); + break; + case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + XXH64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize, /* flags */ 0); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize, /* flags */ 0); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/ + unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + + op = op ? op + flushedSize : op; + + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/deps/zstd/lib/decompress/zstd_decompress_block.c b/deps/zstd/lib/decompress/zstd_decompress_block.c new file mode 100644 index 0000000000..09896a931e --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_decompress_block.c @@ -0,0 +1,2192 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) + { + /* room for litbuffer to fit without read faulting */ + dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } + else if (litSize > ZSTD_LITBUFFEREXTRASIZE) + { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } + else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + } + else + { + /* fits entirely within litExtraBuffer, so no split is necessary */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity, const streaming_operation streaming); +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } else { + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what empirically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + /* + * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be + * loaded in one operation and extracted its fields by simply shifting or + * bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + } + + return seq; +} + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { + seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) { + size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + } + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (--nbSeq) + BIT_reloadDStream(&(seqState.DStream)); + } + } + } + + if (nbSeq > 0) /* there is remaining lit from extra buffer */ + { + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for (; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE size_t +ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) + { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) + { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + } + + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } +} + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX. + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX)); + size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); + + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } + } + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/deps/zstd/lib/decompress/zstd_decompress_block.h b/deps/zstd/lib/decompress/zstd_decompress_block.h new file mode 100644 index 0000000000..9d1318882d --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_decompress_block.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/deps/zstd/lib/decompress/zstd_decompress_internal.h b/deps/zstd/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 0000000000..c2ec5d9fbe --- /dev/null +++ b/deps/zstd/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/deps/zstd/lib/zdict.h b/deps/zstd/lib/zdict.h new file mode 100644 index 0000000000..2268f948a5 --- /dev/null +++ b/deps/zstd/lib/zdict.h @@ -0,0 +1,474 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBLE +# endif +#endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBLE +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=20, and accel=1. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: Dictionary training will fail if there are not enough samples to construct a + * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit). + * If dictionary training fails, you should use zstd without a dictionary, as the dictionary + * would've been ineffective anyways. If you believe your samples would benefit from a dictionary + * please open an issue with details, and we can look into it. + * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); + +typedef struct { + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ +} ZDICT_params_t; + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics according to the zstd + * dictionary format. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each + * sample in order. The samples are used to construct the statistics, so they + * should be representative of what you will compress with this dictionary. + * + * The compression level can be set in `parameters`. You should pass the + * compression level you expect to use in production. The statistics for each + * compression level differ, so tuning the dictionary for the compression level + * can help quite a bit. + * + * You can set an explicit dictionary ID in `parameters`, or allow us to pick + * a random dictionary ID for you, but we can't guarantee no collisions. + * + * The dstDictBuffer and the dictContent may overlap, and the content will be + * appended to the end of the header. If the header + the content doesn't fit in + * maxDictSize the beginning of the content is truncated to make room, since it + * is presumed that the most profitable content is at the end of the dictionary, + * since that is the cheapest to reference. + * + * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * + * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if + * instructed to, using notificationLevel>0. + * NOTE: This function currently may fail in several edge cases including: + * * Not enough samples + * * Samples are uncompressible + * * Samples are all exactly the same + */ +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + +#endif /* ZSTD_ZDICT_H */ + +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +#define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 + +/*! ZDICT_cover_params_t: + * k and d are the only required parameters. + * For others, value 0 means default. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters); + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 20 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t parameters); + + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +#endif /* ZSTD_ZDICT_H_STATIC */ + +#if defined (__cplusplus) +} +#endif diff --git a/deps/zstd/lib/zstd.h b/deps/zstd/lib/zstd.h new file mode 100644 index 0000000000..e5c3f8b68b --- /dev/null +++ b/deps/zstd/lib/zstd.h @@ -0,0 +1,3020 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependencies ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 5 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is re-used, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2(). + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences( ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * + * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t ZSTD_sequenceProducer_F ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F* sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/deps/zstd/lib/zstd_errors.h b/deps/zstd/lib/zstd_errors.h new file mode 100644 index 0000000000..dc75eeebad --- /dev/null +++ b/deps/zstd/lib/zstd_errors.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/griffin/griffin.c b/griffin/griffin.c index 10e75c3ddf..aa2210acd6 100644 --- a/griffin/griffin.c +++ b/griffin/griffin.c @@ -1456,6 +1456,7 @@ DEPENDENCIES #include "../libretro-common/formats/libchdr/libchdr_bitstream.c" #include "../libretro-common/formats/libchdr/libchdr_cdrom.c" #include "../libretro-common/formats/libchdr/libchdr_chd.c" +#include "../libretro-common/formats/libchdr/libchdr_huffman.c" #ifdef HAVE_FLAC #include "../libretro-common/formats/libchdr/libchdr_flac.c" @@ -1466,7 +1467,9 @@ DEPENDENCIES #include "../libretro-common/formats/libchdr/libchdr_lzma.c" #endif -#include "../libretro-common/formats/libchdr/libchdr_huffman.c" +#ifdef HAVE_ZSTD +#include "../libretro-common/formats/libchdr/libchdr_zstd.c" +#endif #include "../libretro-common/streams/chd_stream.c" #endif @@ -1492,6 +1495,18 @@ DEPENDENCIES #include "../deps/7zip/7zStream.c" #endif +#ifdef HAVE_ZSTD +#include "../deps/zstd/lib/common/entropy_common.c" +#include "../deps/zstd/lib/common/error_private.c" +#include "../deps/zstd/lib/common/fse_decompress.c" +#include "../deps/zstd/lib/common/zstd_common.c" +#include "../deps/zstd/lib/common/xxhash.c" +#include "../deps/zstd/lib/decompress/huf_decompress.c" +#include "../deps/zstd/lib/decompress/zstd_ddict.c" +#include "../deps/zstd/lib/decompress/zstd_decompress.c" +#include "../deps/zstd/lib/decompress/zstd_decompress_block.c" +#endif + #ifdef WANT_LIBFAT #include "../deps/libfat/cache.c" #include "../deps/libfat/directory.c" diff --git a/intl/msg_hash_us.h b/intl/msg_hash_us.h index 27d3b740d3..aea45ed317 100644 --- a/intl/msg_hash_us.h +++ b/intl/msg_hash_us.h @@ -878,6 +878,10 @@ MSG_HASH( MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_7ZIP_SUPPORT, "7zip Support" ) +MSG_HASH( + MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_ZSTD_SUPPORT, + "ZStandard Support" + ) MSG_HASH( MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_DYLIB_SUPPORT, "Dynamic Library Support" diff --git a/libretro-common/audio/audio_mixer.c b/libretro-common/audio/audio_mixer.c index ceac806e6e..e490052fd1 100644 --- a/libretro-common/audio/audio_mixer.c +++ b/libretro-common/audio/audio_mixer.c @@ -46,7 +46,9 @@ #endif #ifdef HAVE_DR_FLAC +#include #define DR_FLAC_IMPLEMENTATION +#define DRFLAC_API static INLINE #include #endif @@ -717,7 +719,7 @@ static bool audio_mixer_play_flac( void *flac_buffer = NULL; void *resampler_data = NULL; const retro_resampler_t* resamp = NULL; - drflac *dr_flac = drflac_open_memory((const unsigned char*)sound->types.flac.data,sound->types.flac.size); + drflac *dr_flac = drflac_open_memory((const unsigned char*)sound->types.flac.data, sound->types.flac.size, NULL); if (!dr_flac) return false; @@ -1200,7 +1202,7 @@ static void audio_mixer_mix_flac(float* buffer, size_t num_frames, if (voice->types.flac.position == voice->types.flac.samples) { again: - temp_samples = (unsigned)drflac_read_f32( voice->types.flac.stream, AUDIO_MIXER_TEMP_BUFFER, temp_buffer); + temp_samples = (unsigned)drflac_read_pcm_frames_f32( voice->types.flac.stream, AUDIO_MIXER_TEMP_BUFFER, temp_buffer); if (temp_samples == 0) { if (voice->repeat) @@ -1208,7 +1210,7 @@ again: if (voice->stop_cb) voice->stop_cb(voice->sound, AUDIO_MIXER_SOUND_REPEATED); - drflac_seek_to_sample(voice->types.flac.stream,0); + drflac_seek_to_pcm_frame(voice->types.flac.stream,0); goto again; } diff --git a/libretro-common/formats/libchdr/libchdr_bitstream.c b/libretro-common/formats/libchdr/libchdr_bitstream.c index 1a6cf73ba2..c82a67ddc5 100644 --- a/libretro-common/formats/libchdr/libchdr_bitstream.c +++ b/libretro-common/formats/libchdr/libchdr_bitstream.c @@ -34,6 +34,7 @@ struct bitstream* create_bitstream(const void *src, uint32_t srclength) return bitstream; } + /*----------------------------------------------------- * bitstream_peek - fetch the requested number of bits * but don't advance the input pointer @@ -61,6 +62,7 @@ uint32_t bitstream_peek(struct bitstream* bitstream, int numbits) return bitstream->buffer >> (32 - numbits); } + /*----------------------------------------------------- * bitstream_remove - advance the input pointer by the * specified number of bits @@ -73,6 +75,7 @@ void bitstream_remove(struct bitstream* bitstream, int numbits) bitstream->bits -= numbits; } + /*----------------------------------------------------- * bitstream_read - fetch the requested number of bits *----------------------------------------------------- @@ -85,6 +88,7 @@ uint32_t bitstream_read(struct bitstream* bitstream, int numbits) return result; } + /*------------------------------------------------- * read_offset - return the current read offset *------------------------------------------------- @@ -102,6 +106,7 @@ uint32_t bitstream_read_offset(struct bitstream* bitstream) return result; } + /*------------------------------------------------- * flush - flush to the nearest byte *------------------------------------------------- @@ -117,3 +122,4 @@ uint32_t bitstream_flush(struct bitstream* bitstream) bitstream->bits = bitstream->buffer = 0; return bitstream->doffset; } + diff --git a/libretro-common/formats/libchdr/libchdr_cdrom.c b/libretro-common/formats/libchdr/libchdr_cdrom.c index a8ef038dbc..082963d4b6 100644 --- a/libretro-common/formats/libchdr/libchdr_cdrom.c +++ b/libretro-common/formats/libchdr/libchdr_cdrom.c @@ -15,15 +15,13 @@ schemes will differ after track 1! ***************************************************************************/ -#ifdef WANT_RAW_DATA_SECTOR -#include #include -#include - #include +#ifdef WANT_RAW_DATA_SECTOR + /*************************************************************************** DEBUGGING ***************************************************************************/ @@ -328,7 +326,7 @@ static INLINE uint8_t ecc_source_byte(const uint8_t *sector, uint32_t offset) void ecc_compute_bytes(const uint8_t *sector, const uint16_t *row, int rowlen, uint8_t *val1, uint8_t *val2) { - int component; + int component; *val1 = *val2 = 0; for (component = 0; component < rowlen; component++) { @@ -354,7 +352,7 @@ void ecc_compute_bytes(const uint8_t *sector, const uint16_t *row, int rowlen, u int ecc_verify(const uint8_t *sector) { - int byte; + int byte; /* first verify P bytes */ for (byte = 0; byte < ECC_P_NUM_BYTES; byte++) { @@ -388,7 +386,7 @@ int ecc_verify(const uint8_t *sector) void ecc_generate(uint8_t *sector) { - int byte; + int byte; /* first verify P bytes */ for (byte = 0; byte < ECC_P_NUM_BYTES; byte++) ecc_compute_bytes(sector, poffsets[byte], ECC_P_COMP, §or[ECC_P_OFFSET + byte], §or[ECC_P_OFFSET + ECC_P_NUM_BYTES + byte]); diff --git a/libretro-common/formats/libchdr/libchdr_chd.c b/libretro-common/formats/libchdr/libchdr_chd.c index 9af680f1e8..3ff254c733 100644 --- a/libretro-common/formats/libchdr/libchdr_chd.c +++ b/libretro-common/formats/libchdr/libchdr_chd.c @@ -38,32 +38,37 @@ ***************************************************************************/ #include -#include #include #include #include +#include + #include -#include #include #include +#include #ifdef HAVE_FLAC #include #endif -#ifdef HAVE_7ZIP -#include -#endif - #ifdef HAVE_ZLIB #include #endif -#include -#include +#ifdef HAVE_7ZIP +#include +#endif -#define TRUE 1 -#define FALSE 0 +#ifdef HAVE_ZSTD +#include +#endif + +#if defined(__PS3__) || defined(__PSL1GHT__) +#define __MACTYPES__ +#endif + +#define SHA1_DIGEST_SIZE 20 /*************************************************************************** DEBUGGING @@ -80,14 +85,13 @@ #define OLD_MAP_ENTRY_SIZE 8 /* V1-V2 */ #define METADATA_HEADER_SIZE 16 /* metadata header size */ -#define CRCMAP_HASH_SIZE 4095 /* number of CRC hashtable entries */ - #define MAP_ENTRY_FLAG_TYPE_MASK 0x0f /* what type of hunk */ #define MAP_ENTRY_FLAG_NO_CRC 0x10 /* no CRC is present */ #define CHD_V1_SECTOR_SIZE 512 /* size of a "sector" in the V1 header */ #define COOKIE_VALUE 0xbaadf00d +#define MAX_ZLIB_ALLOCS 64 #define END_OF_LIST_COOKIE "EndOfListCookie" @@ -159,10 +163,10 @@ enum typedef struct _codec_interface codec_interface; struct _codec_interface { - UINT32 compression; /* type of compression */ + uint32_t compression; /* type of compression */ const char *compname; /* name of the algorithm */ - UINT8 lossy; /* is this a lossy algorithm? */ - chd_error (*init)(void *codec, UINT32 hunkbytes); /* codec initialize */ + uint8_t lossy; /* is this a lossy algorithm? */ + chd_error (*init)(void *codec, uint32_t hunkbytes); /* codec initialize */ void (*free)(void *codec); /* codec free */ chd_error (*decompress)(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); /* decompress data */ chd_error (*config)(void *codec, int param, void *config); /* configure */ @@ -172,31 +176,38 @@ struct _codec_interface typedef struct _map_entry map_entry; struct _map_entry { - UINT64 offset; /* offset within the file of the data */ - UINT32 crc; /* 32-bit CRC of the data */ - UINT32 length; /* length of the data */ - UINT8 flags; /* misc flags */ + uint64_t offset; /* offset within the file of the data */ + uint32_t crc; /* 32-bit CRC of the data */ + uint32_t length; /* length of the data */ + uint8_t flags; /* misc flags */ }; /* a single metadata entry */ typedef struct _metadata_entry metadata_entry; struct _metadata_entry { - UINT64 offset; /* offset within the file of the header */ - UINT64 next; /* offset within the file of the next header */ - UINT64 prev; /* offset within the file of the previous header */ - UINT32 length; /* length of the metadata */ - UINT32 metatag; /* metadata tag */ - UINT8 flags; /* flag bits */ + uint64_t offset; /* offset within the file of the header */ + uint64_t next; /* offset within the file of the next header */ + uint64_t prev; /* offset within the file of the previous header */ + uint32_t length; /* length of the metadata */ + uint32_t metatag; /* metadata tag */ + uint8_t flags; /* flag bits */ }; + +typedef struct _huff_codec_data huff_codec_data; +struct _huff_codec_data +{ + struct huffman_decoder* decoder; +}; + + /* internal representation of an open CHD file */ struct _chd_file { - UINT32 cookie; /* cookie, should equal COOKIE_VALUE */ + uint32_t cookie; /* cookie, should equal COOKIE_VALUE */ - RFILE * file; /* handle to the open core file */ - UINT8 owns_file; /* flag indicating if this file should be closed on chd_close() */ + core_file * file; /* handle to the open core file */ chd_header header; /* header, extracted from file */ chd_file * parent; /* pointer to parent file, or NULL */ @@ -204,59 +215,169 @@ struct _chd_file map_entry * map; /* array of map entries */ #ifdef NEED_CACHE_HUNK - UINT8 * cache; /* hunk cache pointer */ - UINT32 cachehunk; /* index of currently cached hunk */ + uint8_t * cache; /* hunk cache pointer */ + uint32_t cachehunk; /* index of currently cached hunk */ - UINT8 * compare; /* hunk compare pointer */ - UINT32 comparehunk; /* index of current compare data */ + uint8_t * compare; /* hunk compare pointer */ + uint32_t comparehunk; /* index of current compare data */ #endif - UINT8 * compressed; /* pointer to buffer for compressed data */ + uint8_t * compressed; /* pointer to buffer for compressed data */ const codec_interface * codecintf[4]; /* interface to the codec */ + huff_codec_data huff_codec_data; /* huff codec data */ #ifdef HAVE_ZLIB zlib_codec_data zlib_codec_data; /* zlib codec data */ cdzl_codec_data cdzl_codec_data; /* cdzl codec data */ #endif #ifdef HAVE_7ZIP + lzma_codec_data lzma_codec_data; /* lzma codec data */ cdlz_codec_data cdlz_codec_data; /* cdlz codec data */ #endif #ifdef HAVE_FLAC + flac_codec_data flac_codec_data; /* flac codec data */ cdfl_codec_data cdfl_codec_data; /* cdfl codec data */ #endif +#ifdef HAVE_ZSTD + zstd_codec_data zstd_codec_data; /* zstd codec data */ + cdzs_codec_data cdzs_codec_data; /* cdzs codec data */ +#endif #ifdef NEED_CACHE_HUNK - UINT32 maxhunk; /* maximum hunk accessed */ + uint32_t maxhunk; /* maximum hunk accessed */ #endif - UINT8 * file_cache; /* cache of underlying file */ + + uint8_t * file_cache; /* cache of underlying file */ }; + /*************************************************************************** GLOBAL VARIABLES ***************************************************************************/ -static const UINT8 nullmd5[CHD_MD5_BYTES] = { 0 }; -static const UINT8 nullsha1[CHD_SHA1_BYTES] = { 0 }; +static const uint8_t nullmd5[CHD_MD5_BYTES] = { 0 }; +static const uint8_t nullsha1[CHD_SHA1_BYTES] = { 0 }; /*************************************************************************** PROTOTYPES ***************************************************************************/ +/* core_file wrappers over stdio */ +static core_file *core_stdio_fopen(char const *path); +static uint64_t core_stdio_fsize(core_file *file); +static size_t core_stdio_fread(void *ptr, size_t size, size_t nmemb, core_file *file); +static int core_stdio_fclose(core_file *file); +static int core_stdio_fclose_nonowner(core_file *file); /* alternate fclose used by chd_open_file */ +static int core_stdio_fseek(core_file* file, int64_t offset, int whence); + /* internal header operations */ static chd_error header_validate(const chd_header *header); static chd_error header_read(chd_file *chd, chd_header *header); /* internal hunk read/write */ #ifdef NEED_CACHE_HUNK -static chd_error hunk_read_into_cache(chd_file *chd, UINT32 hunknum); +static chd_error hunk_read_into_cache(chd_file *chd, uint32_t hunknum); #endif -static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *dest); +static chd_error hunk_read_into_memory(chd_file *chd, uint32_t hunknum, uint8_t *dest); /* internal map access */ static chd_error map_read(chd_file *chd); /* metadata management */ -static chd_error metadata_find_entry(chd_file *chd, UINT32 metatag, UINT32 metaindex, metadata_entry *metaentry); +static chd_error metadata_find_entry(chd_file *chd, uint32_t metatag, uint32_t metaindex, metadata_entry *metaentry); + +/* zlib compression codec */ +chd_error zlib_codec_init(void *codec, uint32_t hunkbytes); +void zlib_codec_free(void *codec); +chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); +voidpf zlib_fast_alloc(voidpf opaque, uInt items, uInt size); +void zlib_fast_free(voidpf opaque, voidpf address); +void zlib_allocator_free(voidpf opaque); + +/* lzma compression codec */ +chd_error lzma_codec_init(void *codec, uint32_t hunkbytes); +void lzma_codec_free(void *codec); +chd_error lzma_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* huff compression codec */ +static chd_error huff_codec_init(void *codec, uint32_t hunkbytes); +static void huff_codec_free(void *codec); +static chd_error huff_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* flac compression codec */ +chd_error flac_codec_init(void *codec, uint32_t hunkbytes); +void flac_codec_free(void *codec); +chd_error flac_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* zstd compression codec */ +chd_error zstd_codec_init(void *codec, uint32_t hunkbytes); +void zstd_codec_free(void *codec); +chd_error zstd_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* cdzl compression codec */ +chd_error cdzl_codec_init(void* codec, uint32_t hunkbytes); +void cdzl_codec_free(void* codec); +chd_error cdzl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* cdlz compression codec */ +chd_error cdlz_codec_init(void* codec, uint32_t hunkbytes); +void cdlz_codec_free(void* codec); +chd_error cdlz_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* cdfl compression codec */ +chd_error cdfl_codec_init(void* codec, uint32_t hunkbytes); +void cdfl_codec_free(void* codec); +chd_error cdfl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + +/* cdzs compression codec */ +chd_error cdzs_codec_init(void *codec, uint32_t hunkbytes); +void cdzs_codec_free(void *codec); +chd_error cdzs_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); + + +/*************************************************************************** + * HUFFMAN DECOMPRESSOR + *************************************************************************** + */ + +static chd_error huff_codec_init(void* codec, uint32_t hunkbytes) +{ + huff_codec_data* huff_codec = (huff_codec_data*) codec; + huff_codec->decoder = create_huffman_decoder(256, 16); + return CHDERR_NONE; +} + +static void huff_codec_free(void *codec) +{ + huff_codec_data* huff_codec = (huff_codec_data*) codec; + delete_huffman_decoder(huff_codec->decoder); +} + +static chd_error huff_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + uint32_t cur; + chd_error result; + huff_codec_data* huff_codec = (huff_codec_data*) codec; + struct bitstream* bitbuf = create_bitstream(src, complen); + + /* first import the tree */ + enum huffman_error err = huffman_import_tree_huffman(huff_codec->decoder, bitbuf); + if (err != HUFFERR_NONE) + { + free(bitbuf); + return CHDERR_DECOMPRESSION_ERROR; + } + + /* then decode the data */ + for (cur = 0; cur < destlen; cur++) + dest[cur] = huffman_decode_one(huff_codec->decoder, bitbuf); + bitstream_flush(bitbuf); + result = bitstream_overflow(bitbuf) ? CHDERR_DECOMPRESSION_ERROR : CHDERR_NONE; + + free(bitbuf); + return result; +} + /*************************************************************************** CODEC INTERFACES @@ -298,7 +419,7 @@ static const codec_interface codec_interfaces[] = NULL }, - /* V5 zlib compression */ + /* V5 zlib compression */ { CHD_CODEC_ZLIB, "zlib (Deflate)", @@ -308,7 +429,6 @@ static const codec_interface codec_interfaces[] = zlib_codec_decompress, NULL }, - /* V5 CD zlib compression */ { CHD_CODEC_CD_ZLIB, @@ -320,8 +440,17 @@ static const codec_interface codec_interfaces[] = NULL }, #endif - #ifdef HAVE_7ZIP + /* V5 lzma compression */ + { + CHD_CODEC_LZMA, + "lzma (LZMA)", + FALSE, + lzma_codec_init, + lzma_codec_free, + lzma_codec_decompress, + NULL + }, /* V5 CD lzma compression */ { CHD_CODEC_CD_LZMA, @@ -333,8 +462,27 @@ static const codec_interface codec_interfaces[] = NULL }, #endif - + /* V5 huffman compression */ + { + CHD_CODEC_HUFFMAN, + "Huffman", + FALSE, + huff_codec_init, + huff_codec_free, + huff_codec_decompress, + NULL + }, #ifdef HAVE_FLAC + /* V5 flac compression */ + { + CHD_CODEC_FLAC, + "flac (FLAC)", + FALSE, + flac_codec_init, + flac_codec_free, + flac_codec_decompress, + NULL + }, /* V5 CD flac compression */ { CHD_CODEC_CD_FLAC, @@ -346,6 +494,29 @@ static const codec_interface codec_interfaces[] = NULL }, #endif +#ifdef HAVE_ZSTD + /* V5 zstd compression */ + { + CHD_CODEC_ZSTD, + "ZStandard", + FALSE, + zstd_codec_init, + zstd_codec_free, + zstd_codec_decompress, + NULL + }, + /* V5 CD zstd compression */ + { + CHD_CODEC_CD_ZSTD, + "cdzs (CD ZStandard)", + FALSE, + cdzs_codec_init, + cdzs_codec_free, + cdzs_codec_decompress, + NULL + } +#endif + }; /*************************************************************************** @@ -353,22 +524,22 @@ static const codec_interface codec_interfaces[] = ***************************************************************************/ /*------------------------------------------------- - get_bigendian_uint64 - fetch a UINT64 from + get_bigendian_uint64_t - fetch a uint64_t from the data stream in bigendian order -------------------------------------------------*/ -static INLINE UINT64 get_bigendian_uint64(const UINT8 *base) +static INLINE uint64_t get_bigendian_uint64_t(const uint8_t *base) { - return ((UINT64)base[0] << 56) | ((UINT64)base[1] << 48) | ((UINT64)base[2] << 40) | ((UINT64)base[3] << 32) | - ((UINT64)base[4] << 24) | ((UINT64)base[5] << 16) | ((UINT64)base[6] << 8) | (UINT64)base[7]; + return ((uint64_t)base[0] << 56) | ((uint64_t)base[1] << 48) | ((uint64_t)base[2] << 40) | ((uint64_t)base[3] << 32) | + ((uint64_t)base[4] << 24) | ((uint64_t)base[5] << 16) | ((uint64_t)base[6] << 8) | (uint64_t)base[7]; } /*------------------------------------------------- - put_bigendian_uint64 - write a UINT64 to + put_bigendian_uint64_t - write a uint64_t to the data stream in bigendian order -------------------------------------------------*/ -static INLINE void put_bigendian_uint64(UINT8 *base, UINT64 value) +static INLINE void put_bigendian_uint64_t(uint8_t *base, uint64_t value) { base[0] = value >> 56; base[1] = value >> 48; @@ -385,10 +556,10 @@ static INLINE void put_bigendian_uint64(UINT8 *base, UINT64 value) the data stream in bigendian order -------------------------------------------------*/ -static INLINE UINT64 get_bigendian_uint48(const UINT8 *base) +static INLINE uint64_t get_bigendian_uint48(const uint8_t *base) { - return ((UINT64)base[0] << 40) | ((UINT64)base[1] << 32) | - ((UINT64)base[2] << 24) | ((UINT64)base[3] << 16) | ((UINT64)base[4] << 8) | (UINT64)base[5]; + return ((uint64_t)base[0] << 40) | ((uint64_t)base[1] << 32) | + ((uint64_t)base[2] << 24) | ((uint64_t)base[3] << 16) | ((uint64_t)base[4] << 8) | (uint64_t)base[5]; } /*------------------------------------------------- @@ -396,7 +567,7 @@ static INLINE UINT64 get_bigendian_uint48(const UINT8 *base) the data stream in bigendian order -------------------------------------------------*/ -static INLINE void put_bigendian_uint48(UINT8 *base, UINT64 value) +static INLINE void put_bigendian_uint48(uint8_t *base, uint64_t value) { value &= 0xffffffffffff; base[0] = value >> 40; @@ -407,36 +578,34 @@ static INLINE void put_bigendian_uint48(UINT8 *base, UINT64 value) base[5] = value; } /*------------------------------------------------- - get_bigendian_uint32 - fetch a UINT32 from + get_bigendian_uint32_t - fetch a uint32_t from the data stream in bigendian order -------------------------------------------------*/ -static INLINE UINT32 get_bigendian_uint32(const UINT8 *base) +static INLINE uint32_t get_bigendian_uint32_t(const uint8_t *base) { return (base[0] << 24) | (base[1] << 16) | (base[2] << 8) | base[3]; } /*------------------------------------------------- - put_bigendian_uint32 - write a UINT32 to + put_bigendian_uint32_t - write a uint32_t to the data stream in bigendian order -------------------------------------------------*/ -#if 0 -static INLINE void put_bigendian_uint32(UINT8 *base, UINT32 value) +static INLINE void put_bigendian_uint32_t(uint8_t *base, uint32_t value) { - base[0] = value >> 24; + base[0] = value >> 24; base[1] = value >> 16; base[2] = value >> 8; base[3] = value; } -#endif /*------------------------------------------------- put_bigendian_uint24 - write a UINT24 to the data stream in bigendian order -------------------------------------------------*/ -static INLINE void put_bigendian_uint24(UINT8 *base, UINT32 value) +static INLINE void put_bigendian_uint24(uint8_t *base, uint32_t value) { value &= 0xffffff; base[0] = value >> 16; @@ -449,27 +618,27 @@ static INLINE void put_bigendian_uint24(UINT8 *base, UINT32 value) the data stream in bigendian order -------------------------------------------------*/ -static INLINE UINT32 get_bigendian_uint24(const UINT8 *base) +static INLINE uint32_t get_bigendian_uint24(const uint8_t *base) { return (base[0] << 16) | (base[1] << 8) | base[2]; } /*------------------------------------------------- - get_bigendian_uint16 - fetch a UINT16 from + get_bigendian_uint16 - fetch a uint16_t from the data stream in bigendian order -------------------------------------------------*/ -static INLINE UINT16 get_bigendian_uint16(const UINT8 *base) +static INLINE uint16_t get_bigendian_uint16(const uint8_t *base) { return (base[0] << 8) | base[1]; } /*------------------------------------------------- - put_bigendian_uint16 - write a UINT16 to + put_bigendian_uint16 - write a uint16_t to the data stream in bigendian order -------------------------------------------------*/ -static INLINE void put_bigendian_uint16(UINT8 *base, UINT16 value) +static INLINE void put_bigendian_uint16(uint8_t *base, uint16_t value) { base[0] = value >> 8; base[1] = value; @@ -480,10 +649,10 @@ static INLINE void put_bigendian_uint16(UINT8 *base, UINT16 value) entry from the datastream -------------------------------------------------*/ -static INLINE void map_extract(const UINT8 *base, map_entry *entry) +static INLINE void map_extract(const uint8_t *base, map_entry *entry) { - entry->offset = get_bigendian_uint64(&base[0]); - entry->crc = get_bigendian_uint32(&base[8]); + entry->offset = get_bigendian_uint64_t(&base[0]); + entry->crc = get_bigendian_uint32_t(&base[8]); entry->length = get_bigendian_uint16(&base[12]) | (base[14] << 16); entry->flags = base[15]; } @@ -493,16 +662,14 @@ static INLINE void map_extract(const UINT8 *base, map_entry *entry) entry to the datastream -------------------------------------------------*/ -#if 0 -static INLINE void map_assemble(UINT8 *base, map_entry *entry) +static INLINE void map_assemble(uint8_t *base, map_entry *entry) { - put_bigendian_uint64(&base[0], entry->offset); - put_bigendian_uint32(&base[8], entry->crc); + put_bigendian_uint64_t(&base[0], entry->offset); + put_bigendian_uint32_t(&base[8], entry->crc); put_bigendian_uint16(&base[12], entry->length); base[14] = entry->length >> 16; base[15] = entry->flags; } -#endif /*------------------------------------------------- map_size_v5 - calculate CHDv5 map size @@ -515,7 +682,7 @@ static INLINE int map_size_v5(chd_header* header) /*------------------------------------------------- crc16 - calculate CRC16 (from hashing.cpp) -------------------------------------------------*/ -static uint16_t crc16(const void *data, uint32_t length) +uint16_t crc16(const void *data, uint32_t length) { uint16_t crc = 0xffff; @@ -563,38 +730,52 @@ static uint16_t crc16(const void *data, uint32_t length) return crc; } +/*------------------------------------------------- + compressed - test if CHD file is compressed ++-------------------------------------------------*/ +static INLINE int chd_compressed(chd_header* header) { + return header->compression[0] != CHD_CODEC_NONE; +} + /*------------------------------------------------- decompress_v5_map - decompress the v5 map -------------------------------------------------*/ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) { - uint8_t rawbuf[16]; - uint16_t mapcrc; - uint32_t mapbytes; - uint64_t firstoffs; + uint32_t hunknum; + int repcount = 0; + uint8_t lastcomp = 0; uint32_t last_self = 0; uint64_t last_parent = 0; - uint8_t lastcomp = 0; - int hunknum, repcount = 0; - enum huffman_error err; - uint8_t lengthbits, selfbits, parentbits; - uint8_t* compressed; - struct bitstream* bitbuf; - struct huffman_decoder* decoder; - uint64_t curoffset; - if (header->mapoffset == 0) + struct bitstream* bitbuf; + uint32_t mapbytes; + uint64_t firstoffs; + uint16_t mapcrc; + uint8_t lengthbits; + uint8_t selfbits; + uint8_t parentbits; + uint8_t *compressed_ptr; + uint8_t rawbuf[16]; + struct huffman_decoder* decoder; + enum huffman_error err; + uint64_t curoffset; + int rawmapsize = map_size_v5(header); + + if (!chd_compressed(header)) { -#if 0 - memset(header->rawmap, 0xff,map_size_v5(header)); -#endif - return CHDERR_READ_ERROR; + header->rawmap = (uint8_t*)malloc(rawmapsize); + if (header->rawmap == NULL) + return CHDERR_OUT_OF_MEMORY; + core_fseek(chd->file, header->mapoffset, SEEK_SET); + core_fread(chd->file, header->rawmap, rawmapsize); + return CHDERR_NONE; } /* read the reader */ - filestream_seek(chd->file, header->mapoffset, SEEK_SET); - filestream_read(chd->file, rawbuf, sizeof(rawbuf)); - mapbytes = get_bigendian_uint32(&rawbuf[0]); + core_fseek(chd->file, header->mapoffset, SEEK_SET); + core_fread(chd->file, rawbuf, sizeof(rawbuf)); + mapbytes = get_bigendian_uint32_t(&rawbuf[0]); firstoffs = get_bigendian_uint48(&rawbuf[4]); mapcrc = get_bigendian_uint16(&rawbuf[10]); lengthbits = rawbuf[12]; @@ -602,23 +783,16 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) parentbits = rawbuf[14]; /* now read the map */ - compressed = (uint8_t*)malloc(sizeof(uint8_t) * mapbytes); - if (compressed == NULL) + compressed_ptr = (uint8_t*)malloc(sizeof(uint8_t) * mapbytes); + if (compressed_ptr == NULL) return CHDERR_OUT_OF_MEMORY; - - filestream_seek(chd->file, header->mapoffset + 16, SEEK_SET); - filestream_read(chd->file, compressed, mapbytes); - bitbuf = create_bitstream(compressed, sizeof(uint8_t) * mapbytes); - if (bitbuf == NULL) - { - free(compressed); - return CHDERR_OUT_OF_MEMORY; - } - - header->rawmap = (uint8_t*)malloc(sizeof(uint8_t) * map_size_v5(header)); + core_fseek(chd->file, header->mapoffset + 16, SEEK_SET); + core_fread(chd->file, compressed_ptr, mapbytes); + bitbuf = create_bitstream(compressed_ptr, sizeof(uint8_t) * mapbytes); + header->rawmap = (uint8_t*)malloc(rawmapsize); if (header->rawmap == NULL) { - free(compressed); + free(compressed_ptr); free(bitbuf); return CHDERR_OUT_OF_MEMORY; } @@ -627,7 +801,7 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) decoder = create_huffman_decoder(16, 8); if (decoder == NULL) { - free(compressed); + free(compressed_ptr); free(bitbuf); return CHDERR_OUT_OF_MEMORY; } @@ -635,34 +809,24 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) err = huffman_import_tree_rle(decoder, bitbuf); if (err != HUFFERR_NONE) { - free(compressed); + free(compressed_ptr); free(bitbuf); delete_huffman_decoder(decoder); return CHDERR_DECOMPRESSION_ERROR; } - for (hunknum = 0; hunknum < (int)header->hunkcount; hunknum++) + for (hunknum = 0; hunknum < header->hunkcount; hunknum++) { uint8_t *rawmap = header->rawmap + (hunknum * 12); if (repcount > 0) - { - rawmap[0] = lastcomp; - repcount--; - } + rawmap[0] = lastcomp, repcount--; else { uint8_t val = huffman_decode_one(decoder, bitbuf); if (val == COMPRESSION_RLE_SMALL) - { - rawmap[0] = lastcomp; - repcount = 2 + huffman_decode_one(decoder, bitbuf); - } + rawmap[0] = lastcomp, repcount = 2 + huffman_decode_one(decoder, bitbuf); else if (val == COMPRESSION_RLE_LARGE) - { - rawmap[0] = lastcomp; - repcount = 2 + 16 + (huffman_decode_one(decoder, bitbuf) << 4); - repcount += huffman_decode_one(decoder, bitbuf); - } + rawmap[0] = lastcomp, repcount = 2 + 16 + (huffman_decode_one(decoder, bitbuf) << 4), repcount += huffman_decode_one(decoder, bitbuf); else rawmap[0] = lastcomp = val; } @@ -670,7 +834,7 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) /* then iterate through the hunks and extract the needed data */ curoffset = firstoffs; - for (hunknum = 0; hunknum < (int)header->hunkcount; hunknum++) + for (hunknum = 0; hunknum < header->hunkcount; hunknum++) { uint8_t *rawmap = header->rawmap + (hunknum * 12); uint64_t offset = curoffset; @@ -732,7 +896,7 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) } /* free memory */ - free(compressed); + free(compressed_ptr); free(bitbuf); delete_huffman_decoder(decoder); @@ -748,9 +912,9 @@ static chd_error decompress_v5_map(chd_file* chd, chd_header* header) entry in old format from the datastream -------------------------------------------------*/ -static INLINE void map_extract_old(const UINT8 *base, map_entry *entry, UINT32 hunkbytes) +static INLINE void map_extract_old(const uint8_t *base, map_entry *entry, uint32_t hunkbytes) { - entry->offset = get_bigendian_uint64(&base[0]); + entry->offset = get_bigendian_uint64_t(&base[0]); entry->crc = 0; entry->length = entry->offset >> 44; entry->flags = MAP_ENTRY_FLAG_NO_CRC | ((entry->length == hunkbytes) ? V34_MAP_ENTRY_TYPE_UNCOMPRESSED : V34_MAP_ENTRY_TYPE_COMPRESSED); @@ -769,11 +933,28 @@ static INLINE void map_extract_old(const UINT8 *base, map_entry *entry, UINT32 h chd_open_file - open a CHD file for access -------------------------------------------------*/ -chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd) +CHD_EXPORT chd_error chd_open_file(FILE *file, int mode, chd_file *parent, chd_file **chd) { + core_file *stream = malloc(sizeof(core_file)); + if (!stream) + return CHDERR_OUT_OF_MEMORY; + stream->argp = file; + stream->fsize = core_stdio_fsize; + stream->fread = core_stdio_fread; + stream->fclose = core_stdio_fclose_nonowner; + stream->fseek = core_stdio_fseek; + + return chd_open_core_file(stream, mode, parent, chd); +} + +/*------------------------------------------------- + chd_open_core_file - open a CHD file for access +-------------------------------------------------*/ + +CHD_EXPORT chd_error chd_open_core_file(core_file *file, int mode, chd_file *parent, chd_file **chd) { chd_file *newchd = NULL; chd_error err; - int intfnum; + size_t intfnum; /* verify parameters */ if (file == NULL) @@ -811,8 +992,15 @@ chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd) EARLY_EXIT(err = CHDERR_UNSUPPORTED_VERSION); /* if we need a parent, make sure we have one */ - if (parent == NULL && (newchd->header.flags & CHDFLAGS_HAS_PARENT)) - EARLY_EXIT(err = CHDERR_REQUIRES_PARENT); + if (parent == NULL) + { + /* Detect parent requirement for versions below 5 */ + if (newchd->header.version < 5 && newchd->header.flags & CHDFLAGS_HAS_PARENT) + EARLY_EXIT(err = CHDERR_REQUIRES_PARENT); + /* Detection for version 5 and above - if parentsha1 != 0, we have a parent */ + else if (newchd->header.version >= 5 && memcmp(nullsha1, newchd->header.parentsha1, sizeof(newchd->header.parentsha1)) != 0) + EARLY_EXIT(err = CHDERR_REQUIRES_PARENT); + } /* make sure we have a valid parent */ if (parent != NULL) @@ -834,6 +1022,8 @@ chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd) if (newchd->header.version < 5) { err = map_read(newchd); + if (err != CHDERR_NONE) + EARLY_EXIT(err); } else { @@ -844,8 +1034,8 @@ chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd) #ifdef NEED_CACHE_HUNK /* allocate and init the hunk cache */ - newchd->cache = (UINT8 *)malloc(newchd->header.hunkbytes); - newchd->compare = (UINT8 *)malloc(newchd->header.hunkbytes); + newchd->cache = (uint8_t *)malloc(newchd->header.hunkbytes); + newchd->compare = (uint8_t *)malloc(newchd->header.hunkbytes); if (newchd->cache == NULL || newchd->compare == NULL) EARLY_EXIT(err = CHDERR_OUT_OF_MEMORY); newchd->cachehunk = ~0; @@ -853,96 +1043,123 @@ chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd) #endif /* allocate the temporary compressed buffer */ - newchd->compressed = (UINT8 *)malloc(newchd->header.hunkbytes); + newchd->compressed = (uint8_t *)malloc(newchd->header.hunkbytes); if (newchd->compressed == NULL) EARLY_EXIT(err = CHDERR_OUT_OF_MEMORY); /* find the codec interface */ if (newchd->header.version < 5) { - for (intfnum = 0; intfnum < (int)ARRAY_SIZE(codec_interfaces); intfnum++) + for (intfnum = 0; intfnum < ARRAY_LENGTH(codec_interfaces); intfnum++) + { if (codec_interfaces[intfnum].compression == newchd->header.compression[0]) { newchd->codecintf[0] = &codec_interfaces[intfnum]; break; } - if (intfnum == ARRAY_SIZE(codec_interfaces)) + } + + if (intfnum == ARRAY_LENGTH(codec_interfaces)) EARLY_EXIT(err = CHDERR_UNSUPPORTED_FORMAT); #ifdef HAVE_ZLIB /* initialize the codec */ if (newchd->codecintf[0]->init != NULL) - { - err = (*newchd->codecintf[0]->init)(&newchd->zlib_codec_data, newchd->header.hunkbytes); - (void)err; - } + { + err = (*newchd->codecintf[0]->init)(&newchd->zlib_codec_data, newchd->header.hunkbytes); + if (err != CHDERR_NONE) + EARLY_EXIT(err); + } #endif } else { - int i, decompnum; + size_t decompnum; /* verify the compression types and initialize the codecs */ - for (decompnum = 0; decompnum < (int)ARRAY_SIZE(newchd->header.compression); decompnum++) + for (decompnum = 0; decompnum < ARRAY_LENGTH(newchd->header.compression); decompnum++) { - for (i = 0 ; i < (int)ARRAY_SIZE(codec_interfaces); i++) + size_t i; + for (i = 0 ; i < ARRAY_LENGTH(codec_interfaces) ; i++) { if (codec_interfaces[i].compression == newchd->header.compression[decompnum]) { newchd->codecintf[decompnum] = &codec_interfaces[i]; - if (newchd->codecintf[decompnum] == NULL && newchd->header.compression[decompnum] != 0) - { - err = CHDERR_UNSUPPORTED_FORMAT; - (void)err; - } - - /* initialize the codec */ - if (newchd->codecintf[decompnum]->init != NULL) - { - void* codec = NULL; - switch (newchd->header.compression[decompnum]) - { - case CHD_CODEC_ZLIB: -#ifdef HAVE_ZLIB - codec = &newchd->zlib_codec_data; -#endif - break; - - case CHD_CODEC_CD_ZLIB: -#ifdef HAVE_ZLIB - codec = &newchd->cdzl_codec_data; -#endif - break; - - case CHD_CODEC_CD_LZMA: -#ifdef HAVE_7ZIP - codec = &newchd->cdlz_codec_data; -#endif - break; - - case CHD_CODEC_CD_FLAC: -#ifdef HAVE_FLAC - codec = &newchd->cdfl_codec_data; -#endif - break; - } - if (codec != NULL) - { - err = (*newchd->codecintf[decompnum]->init)(codec, newchd->header.hunkbytes); - (void)err; - } - } - + break; } } + + if (newchd->codecintf[decompnum] == NULL && newchd->header.compression[decompnum] != 0) + EARLY_EXIT(err = CHDERR_UNSUPPORTED_FORMAT); + + /* initialize the codec */ + if (newchd->codecintf[decompnum]->init != NULL) + { + void* codec = NULL; + switch (newchd->header.compression[decompnum]) + { + case CHD_CODEC_ZLIB: +#ifdef HAVE_ZLIB + codec = &newchd->zlib_codec_data; +#endif + break; + + case CHD_CODEC_LZMA: +#ifdef HAVE_7ZIP + codec = &newchd->lzma_codec_data; +#endif + break; + + case CHD_CODEC_HUFFMAN: + codec = &newchd->huff_codec_data; + break; + + case CHD_CODEC_FLAC: +#ifdef HAVE_FLAC + codec = &newchd->flac_codec_data; +#endif + break; + + case CHD_CODEC_ZSTD: +#ifdef HAVE_ZSTD + codec = &newchd->zstd_codec_data; +#endif + break; + + case CHD_CODEC_CD_ZLIB: +#ifdef HAVE_ZLIB + codec = &newchd->cdzl_codec_data; +#endif + break; + + case CHD_CODEC_CD_LZMA: +#ifdef HAVE_7ZIP + codec = &newchd->cdlz_codec_data; +#endif + break; + + case CHD_CODEC_CD_FLAC: +#ifdef HAVE_FLAC + codec = &newchd->cdfl_codec_data; +#endif + break; + + case CHD_CODEC_CD_ZSTD: +#ifdef HAVE_ZSTD + codec = &newchd->cdzs_codec_data; +#endif + break; + } + + if (codec == NULL) + EARLY_EXIT(err = CHDERR_UNSUPPORTED_FORMAT); + + err = (*newchd->codecintf[decompnum]->init)(codec, newchd->header.hunkbytes); + if (err != CHDERR_NONE) + EARLY_EXIT(err); + } } } -#if 0 - /* HACK */ - if (err != CHDERR_NONE) - EARLY_EXIT(err); -#endif - /* all done */ *chd = newchd; return CHDERR_NONE; @@ -958,21 +1175,21 @@ cleanup: memory -------------------------------------------------*/ -chd_error chd_precache(chd_file *chd) +CHD_EXPORT chd_error chd_precache(chd_file *chd) { - int64_t size, count; + uint64_t count; + uint64_t size; - if (!chd->file_cache) + if (chd->file_cache == NULL) { - filestream_seek(chd->file, 0, SEEK_END); - size = filestream_tell(chd->file); - if (size <= 0) + size = core_fsize(chd->file); + if ((int64_t)size <= 0) return CHDERR_INVALID_DATA; - chd->file_cache = (UINT8*)malloc(size); + chd->file_cache = malloc(size); if (chd->file_cache == NULL) return CHDERR_OUT_OF_MEMORY; - filestream_seek(chd->file, 0, SEEK_SET); - count = filestream_read(chd->file, chd->file_cache, size); + core_fseek(chd->file, 0, SEEK_SET); + count = core_fread(chd->file, chd->file_cache, size); if (count != size) { free(chd->file_cache); @@ -984,16 +1201,21 @@ chd_error chd_precache(chd_file *chd) return CHDERR_NONE; } - /*------------------------------------------------- chd_open - open a CHD file by filename -------------------------------------------------*/ -chd_error chd_open(const char *filename, int mode, chd_file *parent, chd_file **chd) +CHD_EXPORT chd_error chd_open(const char *filename, int mode, chd_file *parent, chd_file **chd) { chd_error err; - RFILE *file = NULL; + core_file *file = NULL; + + if (filename == NULL) + { + err = CHDERR_INVALID_PARAMETER; + goto cleanup; + } /* choose the proper mode */ switch(mode) @@ -1007,27 +1229,19 @@ chd_error chd_open(const char *filename, int mode, chd_file *parent, chd_file ** } /* open the file */ - file = filestream_open(filename, - RETRO_VFS_FILE_ACCESS_READ, - RETRO_VFS_FILE_ACCESS_HINT_NONE); - - if (!file) + file = core_stdio_fopen(filename); + if (file == 0) { err = CHDERR_FILE_NOT_FOUND; goto cleanup; } /* now open the CHD */ - err = chd_open_file(file, mode, parent, chd); - if (err != CHDERR_NONE) - goto cleanup; - - /* we now own this file */ - (*chd)->owns_file = TRUE; + return chd_open_core_file(file, mode, parent, chd); cleanup: if ((err != CHDERR_NONE) && (file != NULL)) - filestream_close(file); + core_fclose(file); return err; } @@ -1035,7 +1249,7 @@ cleanup: chd_close - close a CHD file for access -------------------------------------------------*/ -void chd_close(chd_file *chd) +CHD_EXPORT void chd_close(chd_file *chd) { /* punt if NULL or invalid */ if (chd == NULL || chd->cookie != COOKIE_VALUE) @@ -1051,43 +1265,75 @@ void chd_close(chd_file *chd) } else { - int i; + size_t i; /* Free the codecs */ - for (i = 0 ; i < 4 ; i++) - { - void* codec = NULL; - if (!chd->codecintf[i]) - continue; + for (i = 0 ; i < ARRAY_LENGTH(chd->codecintf); i++) + { + void* codec = NULL; - switch (chd->codecintf[i]->compression) - { - case CHD_CODEC_CD_LZMA: + if (chd->codecintf[i] == NULL) + continue; + + switch (chd->codecintf[i]->compression) + { + case CHD_CODEC_ZLIB: +#ifdef HAVE_ZLIB + codec = &chd->zlib_codec_data; +#endif + break; + + case CHD_CODEC_LZMA: #ifdef HAVE_7ZIP - codec = &chd->cdlz_codec_data; + codec = &chd->lzma_codec_data; #endif - break; + break; - case CHD_CODEC_ZLIB: -#ifdef HAVE_ZLIB - codec = &chd->zlib_codec_data; -#endif - break; + case CHD_CODEC_HUFFMAN: + codec = &chd->huff_codec_data; + break; - case CHD_CODEC_CD_ZLIB: -#ifdef HAVE_ZLIB - codec = &chd->cdzl_codec_data; -#endif - break; - - case CHD_CODEC_CD_FLAC: + case CHD_CODEC_FLAC: #ifdef HAVE_FLAC - codec = &chd->cdfl_codec_data; + codec = &chd->flac_codec_data; #endif - break; - } - if (codec) - (*chd->codecintf[i]->free)(codec); - } + break; + + case CHD_CODEC_ZSTD: +#ifdef HAVE_ZSTD + codec = &chd->zstd_codec_data; +#endif + break; + + case CHD_CODEC_CD_ZLIB: +#ifdef HAVE_ZLIB + codec = &chd->cdzl_codec_data; +#endif + break; + + case CHD_CODEC_CD_LZMA: +#ifdef HAVE_7ZIP + codec = &chd->cdlz_codec_data; +#endif + break; + + case CHD_CODEC_CD_FLAC: +#ifdef HAVE_FLAC + codec = &chd->cdfl_codec_data; +#endif + break; + + case CHD_CODEC_CD_ZSTD: +#ifdef HAVE_ZSTD + codec = &chd->cdzs_codec_data; +#endif + break; + } + + if (codec) + { + (*chd->codecintf[i]->free)(codec); + } + } /* Free the raw map */ if (chd->header.rawmap != NULL) @@ -1111,15 +1357,17 @@ void chd_close(chd_file *chd) free(chd->map); /* close the file */ - if (chd->owns_file && chd->file != NULL) - filestream_close(chd->file); + if (chd->file != NULL) + core_fclose(chd->file); #ifdef NEED_CACHE_HUNK if (PRINTF_MAX_HUNK) printf("Max hunk = %d/%d\n", chd->maxhunk, chd->header.totalhunks); #endif + if (chd->file_cache) + free(chd->file_cache); - if (chd->file_cache) - free(chd->file_cache); + if (chd->parent) + chd_close(chd->parent); /* free our memory */ free(chd); @@ -1130,7 +1378,7 @@ void chd_close(chd_file *chd) core_file -------------------------------------------------*/ -RFILE *chd_core_file(chd_file *chd) +CHD_EXPORT core_file *chd_core_file(chd_file *chd) { return chd->file; } @@ -1140,7 +1388,7 @@ RFILE *chd_core_file(chd_file *chd) the given CHD error -------------------------------------------------*/ -const char *chd_error_string(chd_error err) +CHD_EXPORT const char *chd_error_string(chd_error err) { switch (err) { @@ -1185,7 +1433,7 @@ const char *chd_error_string(chd_error err) extracted header data -------------------------------------------------*/ -const chd_header *chd_get_header(chd_file *chd) +CHD_EXPORT const chd_header *chd_get_header(chd_file *chd) { /* punt if NULL or invalid */ if (chd == NULL || chd->cookie != COOKIE_VALUE) @@ -1194,6 +1442,41 @@ const chd_header *chd_get_header(chd_file *chd) return &chd->header; } +/*------------------------------------------------- + chd_read_header - read CHD header data + from file into the pointed struct +-------------------------------------------------*/ +CHD_EXPORT chd_error chd_read_header(const char *filename, chd_header *header) +{ + chd_error err = CHDERR_NONE; + chd_file *chd = NULL; + + /* punt if NULL */ + if (filename == NULL || header == NULL) + EARLY_EXIT(err = CHDERR_INVALID_PARAMETER); + + /* open the file */ + chd->file = core_stdio_fopen(filename); + if (chd->file == NULL) + EARLY_EXIT(err = CHDERR_FILE_NOT_FOUND); + + /* attempt to read the header */ + err = header_read(chd, header); + if (err != CHDERR_NONE) + EARLY_EXIT(err); + + /* validate the header */ + err = header_validate(header); + if (err != CHDERR_NONE) + EARLY_EXIT(err); + +cleanup: + if (chd->file != NULL) + core_fclose(chd->file); + + return err; +} + /*************************************************************************** CORE DATA READ/WRITE ***************************************************************************/ @@ -1203,14 +1486,18 @@ const chd_header *chd_get_header(chd_file *chd) file -------------------------------------------------*/ -chd_error chd_read(chd_file *chd, UINT32 hunknum, void *buffer) +CHD_EXPORT chd_error chd_read(chd_file *chd, uint32_t hunknum, void *buffer) { /* punt if NULL or invalid */ if (chd == NULL || chd->cookie != COOKIE_VALUE) return CHDERR_INVALID_PARAMETER; + /* if we're past the end, fail */ + if (hunknum >= chd->header.totalhunks) + return CHDERR_HUNK_OUT_OF_RANGE; + /* perform the read */ - return hunk_read_into_memory(chd, hunknum, (UINT8 *)buffer); + return hunk_read_into_memory(chd, hunknum, (uint8_t *)buffer); } /*************************************************************************** @@ -1222,11 +1509,11 @@ chd_error chd_read(chd_file *chd, UINT32 hunknum, void *buffer) of the given type -------------------------------------------------*/ -chd_error chd_get_metadata(chd_file *chd, UINT32 searchtag, UINT32 searchindex, void *output, UINT32 outputlen, UINT32 *resultlen, UINT32 *resulttag, UINT8 *resultflags) +CHD_EXPORT chd_error chd_get_metadata(chd_file *chd, uint32_t searchtag, uint32_t searchindex, void *output, uint32_t outputlen, uint32_t *resultlen, uint32_t *resulttag, uint8_t *resultflags) { metadata_entry metaentry; chd_error err; - int64_t count; + uint32_t count; /* if we didn't find it, just return */ err = metadata_find_entry(chd, searchtag, searchindex, &metaentry); @@ -1236,17 +1523,11 @@ chd_error chd_get_metadata(chd_file *chd, UINT32 searchtag, UINT32 searchindex, if (chd->header.version < 3 && (searchtag == HARD_DISK_METADATA_TAG || searchtag == CHDMETATAG_WILDCARD) && searchindex == 0) { char faux_metadata[256]; - UINT32 faux_length; + uint32_t faux_length; /* fill in the faux metadata */ - snprintf(faux_metadata, - sizeof(faux_metadata), - HARD_DISK_METADATA_FORMAT, - chd->header.obsolete_cylinders, - chd->header.obsolete_heads, - chd->header.obsolete_sectors, - chd->header.hunkbytes / chd->header.obsolete_hunksize); - faux_length = (UINT32)strlen(faux_metadata) + 1; + sprintf(faux_metadata, HARD_DISK_METADATA_FORMAT, chd->header.obsolete_cylinders, chd->header.obsolete_heads, chd->header.obsolete_sectors, chd->header.hunkbytes / chd->header.obsolete_hunksize); + faux_length = (uint32_t)strlen(faux_metadata) + 1; /* copy the metadata itself */ memcpy(output, faux_metadata, MIN(outputlen, faux_length)); @@ -1263,8 +1544,8 @@ chd_error chd_get_metadata(chd_file *chd, UINT32 searchtag, UINT32 searchindex, /* read the metadata */ outputlen = MIN(outputlen, metaentry.length); - filestream_seek(chd->file, metaentry.offset + METADATA_HEADER_SIZE, SEEK_SET); - count = filestream_read(chd->file, output, outputlen); + core_fseek(chd->file, metaentry.offset + METADATA_HEADER_SIZE, SEEK_SET); + count = core_fread(chd->file, output, outputlen); if (count != outputlen) return CHDERR_READ_ERROR; @@ -1287,7 +1568,7 @@ chd_error chd_get_metadata(chd_file *chd, UINT32 searchtag, UINT32 searchindex, parameters -------------------------------------------------*/ -chd_error chd_codec_config(chd_file *chd, int param, void *config) +CHD_EXPORT chd_error chd_codec_config(chd_file *chd, int param, void *config) { return CHDERR_INVALID_PARAMETER; } @@ -1297,7 +1578,7 @@ chd_error chd_codec_config(chd_file *chd, int param, void *config) particular codec -------------------------------------------------*/ -const char *chd_get_codec_name(UINT32 codec) +CHD_EXPORT const char *chd_get_codec_name(uint32_t codec) { return "Unknown"; } @@ -1313,7 +1594,7 @@ const char *chd_get_codec_name(UINT32 codec) static chd_error header_validate(const chd_header *header) { - int intfnum; + size_t intfnum; /* require a valid version */ if (header->version == 0 || header->version > CHD_HEADER_VERSION) @@ -1335,11 +1616,11 @@ static chd_error header_validate(const chd_header *header) return CHDERR_INVALID_PARAMETER; /* require a supported compression mechanism */ - for (intfnum = 0; intfnum < (int)ARRAY_SIZE(codec_interfaces); intfnum++) + for (intfnum = 0; intfnum < ARRAY_LENGTH(codec_interfaces); intfnum++) if (codec_interfaces[intfnum].compression == header->compression[0]) break; - if (intfnum == ARRAY_SIZE(codec_interfaces)) + if (intfnum == ARRAY_LENGTH(codec_interfaces)) return CHDERR_INVALID_PARAMETER; /* require a valid hunksize */ @@ -1375,11 +1656,11 @@ static chd_error header_validate(const chd_header *header) guess at the bytes/unit based on metadata -------------------------------------------------*/ -static UINT32 header_guess_unitbytes(chd_file *chd) +static uint32_t header_guess_unitbytes(chd_file *chd) { /* look for hard disk metadata; if found, then the unit size == sector size */ char metadata[512]; - unsigned int i0, i1, i2, i3; + int i0, i1, i2, i3; if (chd_get_metadata(chd, HARD_DISK_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE && sscanf(metadata, HARD_DISK_METADATA_FORMAT, &i0, &i1, &i2, &i3) == 4) return i3; @@ -1388,7 +1669,7 @@ static UINT32 header_guess_unitbytes(chd_file *chd) if (chd_get_metadata(chd, CDROM_OLD_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE || chd_get_metadata(chd, CDROM_TRACK_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE || chd_get_metadata(chd, CDROM_TRACK_METADATA2_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE || - chd_get_metadata(chd, GDROM_OLD_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE || + chd_get_metadata(chd, GDROM_OLD_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE || chd_get_metadata(chd, GDROM_TRACK_METADATA_TAG, 0, metadata, sizeof(metadata), NULL, NULL, NULL) == CHDERR_NONE) return CD_FRAME_SIZE; @@ -1403,8 +1684,8 @@ static UINT32 header_guess_unitbytes(chd_file *chd) static chd_error header_read(chd_file *chd, chd_header *header) { - UINT8 rawheader[CHD_MAX_HEADER_SIZE]; - int64_t count; + uint8_t rawheader[CHD_MAX_HEADER_SIZE]; + uint32_t count; /* punt if NULL */ if (header == NULL) @@ -1415,8 +1696,8 @@ static chd_error header_read(chd_file *chd, chd_header *header) return CHDERR_INVALID_FILE; /* seek and read */ - filestream_seek(chd->file, 0, SEEK_SET); - count = filestream_read(chd->file, rawheader, sizeof(rawheader)); + core_fseek(chd->file, 0, SEEK_SET); + count = core_fread(chd->file, rawheader, sizeof(rawheader)); if (count != sizeof(rawheader)) return CHDERR_READ_ERROR; @@ -1426,8 +1707,8 @@ static chd_error header_read(chd_file *chd, chd_header *header) /* extract the direct data */ memset(header, 0, sizeof(*header)); - header->length = get_bigendian_uint32(&rawheader[8]); - header->version = get_bigendian_uint32(&rawheader[12]); + header->length = get_bigendian_uint32_t(&rawheader[8]); + header->version = get_bigendian_uint32_t(&rawheader[12]); /* make sure it's a version we understand */ if (header->version == 0 || header->version > CHD_HEADER_VERSION) @@ -1443,23 +1724,28 @@ static chd_error header_read(chd_file *chd, chd_header *header) return CHDERR_INVALID_DATA; /* extract the common data */ - header->flags = get_bigendian_uint32(&rawheader[16]); - header->compression[0] = get_bigendian_uint32(&rawheader[20]); + header->flags = get_bigendian_uint32_t(&rawheader[16]); + header->compression[0] = get_bigendian_uint32_t(&rawheader[20]); + header->compression[1] = CHD_CODEC_NONE; + header->compression[2] = CHD_CODEC_NONE; + header->compression[3] = CHD_CODEC_NONE; /* extract the V1/V2-specific data */ if (header->version < 3) { - int seclen = (header->version == 1) ? CHD_V1_SECTOR_SIZE : get_bigendian_uint32(&rawheader[76]); - header->obsolete_hunksize = get_bigendian_uint32(&rawheader[24]); - header->totalhunks = get_bigendian_uint32(&rawheader[28]); - header->obsolete_cylinders = get_bigendian_uint32(&rawheader[32]); - header->obsolete_heads = get_bigendian_uint32(&rawheader[36]); - header->obsolete_sectors = get_bigendian_uint32(&rawheader[40]); + int seclen = (header->version == 1) ? CHD_V1_SECTOR_SIZE : get_bigendian_uint32_t(&rawheader[76]); + header->obsolete_hunksize = get_bigendian_uint32_t(&rawheader[24]); + header->totalhunks = get_bigendian_uint32_t(&rawheader[28]); + header->obsolete_cylinders = get_bigendian_uint32_t(&rawheader[32]); + header->obsolete_heads = get_bigendian_uint32_t(&rawheader[36]); + header->obsolete_sectors = get_bigendian_uint32_t(&rawheader[40]); memcpy(header->md5, &rawheader[44], CHD_MD5_BYTES); memcpy(header->parentmd5, &rawheader[60], CHD_MD5_BYTES); - header->logicalbytes = (UINT64)header->obsolete_cylinders * (UINT64)header->obsolete_heads * (UINT64)header->obsolete_sectors * (UINT64)seclen; + header->logicalbytes = (uint64_t)header->obsolete_cylinders * (uint64_t)header->obsolete_heads * (uint64_t)header->obsolete_sectors * (uint64_t)seclen; header->hunkbytes = seclen * header->obsolete_hunksize; header->unitbytes = header_guess_unitbytes(chd); + if (header->unitbytes == 0) + return CHDERR_INVALID_DATA; header->unitcount = (header->logicalbytes + header->unitbytes - 1) / header->unitbytes; header->metaoffset = 0; } @@ -1467,13 +1753,15 @@ static chd_error header_read(chd_file *chd, chd_header *header) /* extract the V3-specific data */ else if (header->version == 3) { - header->totalhunks = get_bigendian_uint32(&rawheader[24]); - header->logicalbytes = get_bigendian_uint64(&rawheader[28]); - header->metaoffset = get_bigendian_uint64(&rawheader[36]); + header->totalhunks = get_bigendian_uint32_t(&rawheader[24]); + header->logicalbytes = get_bigendian_uint64_t(&rawheader[28]); + header->metaoffset = get_bigendian_uint64_t(&rawheader[36]); memcpy(header->md5, &rawheader[44], CHD_MD5_BYTES); memcpy(header->parentmd5, &rawheader[60], CHD_MD5_BYTES); - header->hunkbytes = get_bigendian_uint32(&rawheader[76]); + header->hunkbytes = get_bigendian_uint32_t(&rawheader[76]); header->unitbytes = header_guess_unitbytes(chd); + if (header->unitbytes == 0) + return CHDERR_INVALID_DATA; header->unitcount = (header->logicalbytes + header->unitbytes - 1) / header->unitbytes; memcpy(header->sha1, &rawheader[80], CHD_SHA1_BYTES); memcpy(header->parentsha1, &rawheader[100], CHD_SHA1_BYTES); @@ -1482,11 +1770,13 @@ static chd_error header_read(chd_file *chd, chd_header *header) /* extract the V4-specific data */ else if (header->version == 4) { - header->totalhunks = get_bigendian_uint32(&rawheader[24]); - header->logicalbytes = get_bigendian_uint64(&rawheader[28]); - header->metaoffset = get_bigendian_uint64(&rawheader[36]); - header->hunkbytes = get_bigendian_uint32(&rawheader[44]); + header->totalhunks = get_bigendian_uint32_t(&rawheader[24]); + header->logicalbytes = get_bigendian_uint64_t(&rawheader[28]); + header->metaoffset = get_bigendian_uint64_t(&rawheader[36]); + header->hunkbytes = get_bigendian_uint32_t(&rawheader[44]); header->unitbytes = header_guess_unitbytes(chd); + if (header->unitbytes == 0) + return CHDERR_INVALID_DATA; header->unitcount = (header->logicalbytes + header->unitbytes - 1) / header->unitbytes; memcpy(header->sha1, &rawheader[48], CHD_SHA1_BYTES); memcpy(header->parentsha1, &rawheader[68], CHD_SHA1_BYTES); @@ -1497,23 +1787,27 @@ static chd_error header_read(chd_file *chd, chd_header *header) else if (header->version == 5) { /* TODO */ - header->compression[0] = get_bigendian_uint32(&rawheader[16]); - header->compression[1] = get_bigendian_uint32(&rawheader[20]); - header->compression[2] = get_bigendian_uint32(&rawheader[24]); - header->compression[3] = get_bigendian_uint32(&rawheader[28]); - header->logicalbytes = get_bigendian_uint64(&rawheader[32]); - header->mapoffset = get_bigendian_uint64(&rawheader[40]); - header->metaoffset = get_bigendian_uint64(&rawheader[48]); - header->hunkbytes = get_bigendian_uint32(&rawheader[56]); - header->hunkcount = (UINT32)((header->logicalbytes + header->hunkbytes - 1) / header->hunkbytes); - header->unitbytes = get_bigendian_uint32(&rawheader[60]); + header->compression[0] = get_bigendian_uint32_t(&rawheader[16]); + header->compression[1] = get_bigendian_uint32_t(&rawheader[20]); + header->compression[2] = get_bigendian_uint32_t(&rawheader[24]); + header->compression[3] = get_bigendian_uint32_t(&rawheader[28]); + header->logicalbytes = get_bigendian_uint64_t(&rawheader[32]); + header->mapoffset = get_bigendian_uint64_t(&rawheader[40]); + header->metaoffset = get_bigendian_uint64_t(&rawheader[48]); + header->hunkbytes = get_bigendian_uint32_t(&rawheader[56]); + if (header->hunkbytes == 0) + return CHDERR_INVALID_DATA; + header->hunkcount = (header->logicalbytes + header->hunkbytes - 1) / header->hunkbytes; + header->unitbytes = get_bigendian_uint32_t(&rawheader[60]); + if (header->unitbytes == 0) + return CHDERR_INVALID_DATA; header->unitcount = (header->logicalbytes + header->unitbytes - 1) / header->unitbytes; memcpy(header->sha1, &rawheader[84], CHD_SHA1_BYTES); memcpy(header->parentsha1, &rawheader[104], CHD_SHA1_BYTES); memcpy(header->rawsha1, &rawheader[64], CHD_SHA1_BYTES); /* determine properties of map entries */ - header->mapentrybytes = 12; /* TODO compressed() ? 12 : 4; */ + header->mapentrybytes = chd_compressed(header) ? 12 : 4; /* hack */ header->totalhunks = header->hunkcount; @@ -1533,13 +1827,59 @@ static chd_error header_read(chd_file *chd, chd_header *header) INTERNAL HUNK READ/WRITE ***************************************************************************/ +/*------------------------------------------------- + hunk_read_compressed - read a compressed + hunk +-------------------------------------------------*/ + +static uint8_t* hunk_read_compressed(chd_file *chd, uint64_t offset, size_t size) +{ + size_t bytes; + + if (chd->file_cache != NULL) + { + return chd->file_cache + offset; + } + else + { + core_fseek(chd->file, offset, SEEK_SET); + bytes = core_fread(chd->file, chd->compressed, size); + if (bytes != size) + return NULL; + return chd->compressed; + } +} + +/*------------------------------------------------- + hunk_read_uncompressed - read an uncompressed + hunk +-------------------------------------------------*/ + +static chd_error hunk_read_uncompressed(chd_file *chd, uint64_t offset, size_t size, uint8_t *dest) +{ + size_t bytes; + + if (chd->file_cache != NULL) + { + memcpy(dest, chd->file_cache + offset, size); + } + else + { + core_fseek(chd->file, offset, SEEK_SET); + bytes = core_fread(chd->file, dest, size); + if (bytes != size) + return CHDERR_READ_ERROR; + } + return CHDERR_NONE; +} + #ifdef NEED_CACHE_HUNK /*------------------------------------------------- hunk_read_into_cache - read a hunk into the CHD's hunk cache -------------------------------------------------*/ -static chd_error hunk_read_into_cache(chd_file *chd, UINT32 hunknum) +static chd_error hunk_read_into_cache(chd_file *chd, uint32_t hunknum) { chd_error err; @@ -1563,49 +1903,12 @@ static chd_error hunk_read_into_cache(chd_file *chd, UINT32 hunknum) } #endif -/*------------------------------------------------- - hunk_read_compressed - read a compressed - hunk --------------------------------------------------*/ - -static UINT8* hunk_read_compressed(chd_file *chd, UINT64 offset, size_t len) -{ - int64_t bytes; - if (chd->file_cache) - return chd->file_cache + offset; - filestream_seek(chd->file, offset, SEEK_SET); - bytes = filestream_read(chd->file, chd->compressed, len); - if (bytes != (int64_t)len) - return NULL; - return chd->compressed; -} - -/*------------------------------------------------- - hunk_read_uncompressed - read an uncompressed - hunk --------------------------------------------------*/ - -static chd_error hunk_read_uncompressed(chd_file *chd, UINT64 offset, size_t len, UINT8 *dest) -{ - int64_t bytes; - if (chd->file_cache) - { - memcpy(dest, chd->file_cache + offset, len); - return CHDERR_NONE; - } - filestream_seek(chd->file, offset, SEEK_SET); - bytes = filestream_read(chd->file, dest, len); - if (bytes != (int64_t)len) - return CHDERR_READ_ERROR; - return CHDERR_NONE; -} - /*------------------------------------------------- hunk_read_into_memory - read a hunk into memory at the given location -------------------------------------------------*/ -static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *dest) +static chd_error hunk_read_into_memory(chd_file *chd, uint32_t hunknum, uint8_t *dest) { chd_error err; @@ -1623,8 +1926,8 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des if (chd->header.version < 5) { map_entry *entry = &chd->map[hunknum]; - UINT32 bytes; - UINT8* compressed_bytes; + uint32_t bytes; + uint8_t* compressed_bytes; /* switch off the entry type */ switch (entry->flags & MAP_ENTRY_FLAG_TYPE_MASK) @@ -1632,36 +1935,37 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des /* compressed data */ case V34_MAP_ENTRY_TYPE_COMPRESSED: { - /* read it into the decompression buffer */ + void *codec = NULL; - void *codec; - compressed_bytes = hunk_read_compressed(chd, entry->offset, - entry->length); - if (compressed_bytes == NULL) - return CHDERR_READ_ERROR; + /* read it into the decompression buffer */ + compressed_bytes = hunk_read_compressed(chd, entry->offset, entry->length); + if (compressed_bytes == NULL) + { + return CHDERR_READ_ERROR; + } #ifdef HAVE_ZLIB - /* now decompress using the codec */ - err = CHDERR_NONE; - codec = &chd->zlib_codec_data; - if (chd->codecintf[0]->decompress != NULL) - err = (*chd->codecintf[0]->decompress)(codec, compressed_bytes, entry->length, dest, chd->header.hunkbytes); - if (err != CHDERR_NONE) - return err; + /* now decompress using the codec */ + err = CHDERR_NONE; + codec = &chd->zlib_codec_data; + if (chd->codecintf[0]->decompress != NULL) + err = (*chd->codecintf[0]->decompress)(codec, compressed_bytes, entry->length, dest, chd->header.hunkbytes); + if (err != CHDERR_NONE) + return err; #endif - } break; + } /* uncompressed data */ case V34_MAP_ENTRY_TYPE_UNCOMPRESSED: - err = hunk_read_uncompressed(chd, entry->offset, chd->header.hunkbytes, dest); - if (err != CHDERR_NONE) - return err; + err = hunk_read_uncompressed(chd, entry->offset, chd->header.hunkbytes, dest); + if (err != CHDERR_NONE) + return err; break; /* mini-compressed data */ case V34_MAP_ENTRY_TYPE_MINI: - put_bigendian_uint64(&dest[0], entry->offset); + put_bigendian_uint64_t(&dest[0], entry->offset); for (bytes = 8; bytes < chd->header.hunkbytes; bytes++) dest[bytes] = dest[bytes - 8]; break; @@ -1672,11 +1976,11 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des if (chd->cachehunk == entry->offset && dest == chd->cache) break; #endif - return hunk_read_into_memory(chd, (UINT32)entry->offset, dest); + return hunk_read_into_memory(chd, entry->offset, dest); /* parent-referenced data */ case V34_MAP_ENTRY_TYPE_PARENT_HUNK: - err = hunk_read_into_memory(chd->parent, (UINT32)entry->offset, dest); + err = hunk_read_into_memory(chd->parent, entry->offset, dest); if (err != CHDERR_NONE) return err; break; @@ -1693,55 +1997,75 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des uint16_t blockcrc; #endif uint8_t *rawmap = &chd->header.rawmap[chd->header.mapentrybytes * hunknum]; - UINT8 *compressed_bytes; + uint8_t* compressed_bytes; -#if 0 - /* uncompressed case - TODO */ - if (!compressed()) + /* uncompressed case */ + if (!chd_compressed(&chd->header)) { - blockoffs = uint64_t(be_read(rawmap, 4)) * uint64_t(m_hunkbytes); - if (blockoffs != 0) - file_read(blockoffs, dest, m_hunkbytes); + blockoffs = (uint64_t)get_bigendian_uint32_t(rawmap) * (uint64_t)chd->header.hunkbytes; + if (blockoffs != 0) { + core_fseek(chd->file, blockoffs, SEEK_SET); + /*int result =*/ + core_fread(chd->file, dest, chd->header.hunkbytes); + /* TODO else if (m_parent_missing) - throw CHDERR_REQUIRES_PARENT; - else if (m_parent != nullptr) - m_parent->read_hunk(hunknum, dest); - else - memset(dest, 0, m_hunkbytes); + throw CHDERR_REQUIRES_PARENT; */ + } else if (chd->parent) { + err = hunk_read_into_memory(chd->parent, hunknum, dest); + if (err != CHDERR_NONE) + return err; + } else { + memset(dest, 0, chd->header.hunkbytes); + } + return CHDERR_NONE; } -#endif /* compressed case */ - blocklen = get_bigendian_uint24(&rawmap[1]); + blocklen = get_bigendian_uint24(&rawmap[1]); blockoffs = get_bigendian_uint48(&rawmap[4]); #ifdef VERIFY_BLOCK_CRC - blockcrc = get_bigendian_uint16(&rawmap[10]); + blockcrc = get_bigendian_uint16(&rawmap[10]); #endif + codec = NULL; switch (rawmap[0]) { case COMPRESSION_TYPE_0: case COMPRESSION_TYPE_1: case COMPRESSION_TYPE_2: case COMPRESSION_TYPE_3: - compressed_bytes = hunk_read_compressed(chd, blockoffs, blocklen); - if (compressed_bytes == NULL) - return CHDERR_READ_ERROR; - if (!chd->codecintf[rawmap[0]]) - return CHDERR_UNSUPPORTED_FORMAT; + compressed_bytes = hunk_read_compressed(chd, blockoffs, blocklen); + if (compressed_bytes == NULL) + return CHDERR_READ_ERROR; switch (chd->codecintf[rawmap[0]]->compression) { - case CHD_CODEC_CD_LZMA: -#ifdef HAVE_7ZIP - codec = &chd->cdlz_codec_data; + case CHD_CODEC_ZLIB: +#ifdef HAVE_ZLIB + codec = &chd->zlib_codec_data; #endif break; - case CHD_CODEC_ZLIB: -#ifdef HAVE_ZLIB - codec = &chd->zlib_codec_data; + case CHD_CODEC_LZMA: +#ifdef HAVE_7ZIP + codec = &chd->lzma_codec_data; #endif - break; + break; + + case CHD_CODEC_HUFFMAN: + codec = &chd->huff_codec_data; + break; + + case CHD_CODEC_FLAC: +#ifdef HAVE_FLAC + codec = &chd->flac_codec_data; +#endif + break; + + case CHD_CODEC_ZSTD: +#ifdef HAVE_ZSTD + codec = &chd->zstd_codec_data; +#endif + break; case CHD_CODEC_CD_ZLIB: #ifdef HAVE_ZLIB @@ -1749,15 +2073,28 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des #endif break; + case CHD_CODEC_CD_LZMA: +#ifdef HAVE_7ZIP + codec = &chd->cdlz_codec_data; +#endif + break; + case CHD_CODEC_CD_FLAC: #ifdef HAVE_FLAC codec = &chd->cdfl_codec_data; +#endif + break; + + case CHD_CODEC_CD_ZSTD: +#ifdef HAVE_ZSTD + codec = &chd->cdzs_codec_data; #endif break; } + if (codec==NULL) return CHDERR_CODEC_ERROR; - err = (*chd->codecintf[rawmap[0]]->decompress)(codec, compressed_bytes, blocklen, dest, chd->header.hunkbytes); + err = chd->codecintf[rawmap[0]]->decompress(codec, compressed_bytes, blocklen, dest, chd->header.hunkbytes); if (err != CHDERR_NONE) return err; #ifdef VERIFY_BLOCK_CRC @@ -1767,26 +2104,49 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des return CHDERR_NONE; case COMPRESSION_NONE: - err = hunk_read_uncompressed(chd, blockoffs, blocklen, dest); - if (err != CHDERR_NONE) - return err; + err = hunk_read_uncompressed(chd, blockoffs, blocklen, dest); + if (err != CHDERR_NONE) + return err; #ifdef VERIFY_BLOCK_CRC - if (crc16(dest, chd->header.hunkbytes) != blockcrc) - return CHDERR_DECOMPRESSION_ERROR; + if (crc16(dest, chd->header.hunkbytes) != blockcrc) + return CHDERR_DECOMPRESSION_ERROR; #endif return CHDERR_NONE; case COMPRESSION_SELF: - return hunk_read_into_memory(chd, (UINT32)blockoffs, dest); + return hunk_read_into_memory(chd, blockoffs, dest); case COMPRESSION_PARENT: -#if 0 - /* TODO */ - if (m_parent_missing) + { + uint8_t units_in_hunk = chd->header.hunkbytes / chd->header.unitbytes; + + if (chd->parent == NULL) return CHDERR_REQUIRES_PARENT; - return m_parent->read_bytes(uint64_t(blockoffs) * uint64_t(m_parent->unit_bytes()), dest, m_hunkbytes); -#endif - return CHDERR_DECOMPRESSION_ERROR; + + /* blockoffs is aligned to units_in_hunk */ + if (blockoffs % units_in_hunk == 0) { + return hunk_read_into_memory(chd->parent, blockoffs / units_in_hunk, dest); + /* blockoffs is not aligned to units_in_hunk */ + } else { + uint32_t unit_in_hunk = blockoffs % units_in_hunk; + uint8_t *buf = malloc(chd->header.hunkbytes); + /* Read first half of hunk which contains blockoffs */ + err = hunk_read_into_memory(chd->parent, blockoffs / units_in_hunk, buf); + if (err != CHDERR_NONE) { + free(buf); + return err; + } + memcpy(dest, buf + unit_in_hunk * chd->header.unitbytes, (units_in_hunk - unit_in_hunk) * chd->header.unitbytes); + /* Read second half of hunk which contains blockoffs */ + err = hunk_read_into_memory(chd->parent, (blockoffs / units_in_hunk) + 1, buf); + if (err != CHDERR_NONE) { + free(buf); + return err; + } + memcpy(dest + (units_in_hunk - unit_in_hunk) * chd->header.unitbytes, buf, unit_in_hunk * chd->header.unitbytes); + free(buf); + } + } } return CHDERR_NONE; } @@ -1799,28 +2159,19 @@ static chd_error hunk_read_into_memory(chd_file *chd, UINT32 hunknum, UINT8 *des INTERNAL MAP ACCESS ***************************************************************************/ -static size_t core_fsize(RFILE *f) -{ - int64_t rv, p = filestream_tell(f); - filestream_seek(f, 0, SEEK_END); - rv = filestream_tell(f); - filestream_seek(f, p, SEEK_SET); - return rv; -} - /*------------------------------------------------- map_read - read the initial sector map -------------------------------------------------*/ static chd_error map_read(chd_file *chd) { - UINT32 entrysize = (chd->header.version < 3) ? OLD_MAP_ENTRY_SIZE : MAP_ENTRY_SIZE; - UINT8 raw_map_entries[MAP_STACK_ENTRIES * MAP_ENTRY_SIZE]; - UINT64 fileoffset, maxoffset = 0; - UINT8 cookie[MAP_ENTRY_SIZE]; - int64_t count; + uint32_t entrysize = (chd->header.version < 3) ? OLD_MAP_ENTRY_SIZE : MAP_ENTRY_SIZE; + uint8_t raw_map_entries[MAP_STACK_ENTRIES * MAP_ENTRY_SIZE]; + uint64_t fileoffset, maxoffset = 0; + uint8_t cookie[MAP_ENTRY_SIZE]; + uint32_t count; chd_error err; - int i; + uint32_t i; /* first allocate memory */ chd->map = (map_entry *)malloc(sizeof(chd->map[0]) * chd->header.totalhunks); @@ -1829,7 +2180,7 @@ static chd_error map_read(chd_file *chd) /* read the map entries in in chunks and extract to the map list */ fileoffset = chd->header.length; - for (i = 0; i < (int)chd->header.totalhunks; i += MAP_STACK_ENTRIES) + for (i = 0; i < chd->header.totalhunks; i += MAP_STACK_ENTRIES) { /* compute how many entries this time */ int entries = chd->header.totalhunks - i, j; @@ -1837,8 +2188,8 @@ static chd_error map_read(chd_file *chd) entries = MAP_STACK_ENTRIES; /* read that many */ - filestream_seek(chd->file, fileoffset, SEEK_SET); - count = filestream_read(chd->file, raw_map_entries, entries * entrysize); + core_fseek(chd->file, fileoffset, SEEK_SET); + count = core_fread(chd->file, raw_map_entries, entries * entrysize); if (count != entries * entrysize) { err = CHDERR_READ_ERROR; @@ -1866,8 +2217,8 @@ static chd_error map_read(chd_file *chd) } /* verify the cookie */ - filestream_seek(chd->file, fileoffset, SEEK_SET); - count = filestream_read(chd->file, &cookie, entrysize); + core_fseek(chd->file, fileoffset, SEEK_SET); + count = core_fread(chd->file, &cookie, entrysize); if (count != entrysize || memcmp(&cookie, END_OF_LIST_COOKIE, entrysize)) { err = CHDERR_INVALID_FILE; @@ -1897,7 +2248,7 @@ cleanup: metadata_find_entry - find a metadata entry -------------------------------------------------*/ -static chd_error metadata_find_entry(chd_file *chd, UINT32 metatag, UINT32 metaindex, metadata_entry *metaentry) +static chd_error metadata_find_entry(chd_file *chd, uint32_t metatag, uint32_t metaindex, metadata_entry *metaentry) { /* start at the beginning */ metaentry->offset = chd->header.metaoffset; @@ -1906,19 +2257,19 @@ static chd_error metadata_find_entry(chd_file *chd, UINT32 metatag, UINT32 metai /* loop until we run out of options */ while (metaentry->offset != 0) { - UINT8 raw_meta_header[METADATA_HEADER_SIZE]; - int64_t count; + uint8_t raw_meta_header[METADATA_HEADER_SIZE]; + uint32_t count; /* read the raw header */ - filestream_seek(chd->file, metaentry->offset, SEEK_SET); - count = filestream_read(chd->file, raw_meta_header, sizeof(raw_meta_header)); + core_fseek(chd->file, metaentry->offset, SEEK_SET); + count = core_fread(chd->file, raw_meta_header, sizeof(raw_meta_header)); if (count != sizeof(raw_meta_header)) break; /* extract the data */ - metaentry->metatag = get_bigendian_uint32(&raw_meta_header[0]); - metaentry->length = get_bigendian_uint32(&raw_meta_header[4]); - metaentry->next = get_bigendian_uint64(&raw_meta_header[8]); + metaentry->metatag = get_bigendian_uint32_t(&raw_meta_header[0]); + metaentry->length = get_bigendian_uint32_t(&raw_meta_header[4]); + metaentry->next = get_bigendian_uint64_t(&raw_meta_header[8]); /* flags are encoded in the high byte of length */ metaentry->flags = metaentry->length >> 24; @@ -1937,3 +2288,87 @@ static chd_error metadata_find_entry(chd_file *chd, UINT32 metatag, UINT32 metai /* if we get here, we didn't find it */ return CHDERR_METADATA_NOT_FOUND; } + +/*------------------------------------------------- + core_stdio_fopen - core_file wrapper over fopen +-------------------------------------------------*/ +static core_file *core_stdio_fopen(char const *path) { + core_file *file = malloc(sizeof(core_file)); + if (!file) + return NULL; + if (!(file->argp = fopen(path, "rb"))) { + free(file); + return NULL; + } + file->fsize = core_stdio_fsize; + file->fread = core_stdio_fread; + file->fclose = core_stdio_fclose; + file->fseek = core_stdio_fseek; + return file; +} + +/*------------------------------------------------- + core_stdio_fsize - core_file function for + getting file size with stdio +-------------------------------------------------*/ +static uint64_t core_stdio_fsize(core_file *file) { +#if defined USE_LIBRETRO_VFS + #define core_stdio_fseek_impl fseek + #define core_stdio_ftell_impl ftell +#elif defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(__WIN64__) + #define core_stdio_fseek_impl _fseeki64 + #define core_stdio_ftell_impl _ftelli64 +#elif defined(_LARGEFILE_SOURCE) && defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 64 + #define core_stdio_fseek_impl fseeko64 + #define core_stdio_ftell_impl ftello64 +#elif defined(__PS3__) && !defined(__PSL1GHT__) || defined(__SWITCH__) || defined(__vita__) + #define core_stdio_fseek_impl(x,y,z) fseek(x,(off_t)y,z) + #define core_stdio_ftell_impl(x) (off_t)ftell(x) +#else + #define core_stdio_fseek_impl fseeko + #define core_stdio_ftell_impl ftello +#endif + FILE *fp; + uint64_t p, rv; + fp = (FILE*)file->argp; + + p = core_stdio_ftell_impl(fp); + core_stdio_fseek_impl(fp, 0, SEEK_END); + rv = core_stdio_ftell_impl(fp); + core_stdio_fseek_impl(fp, p, SEEK_SET); + return rv; +} + +/*------------------------------------------------- + core_stdio_fread - core_file wrapper over fread +-------------------------------------------------*/ +static size_t core_stdio_fread(void *ptr, size_t size, size_t nmemb, core_file *file) { + return fread(ptr, size, nmemb, (FILE*)file->argp); +} + +/*------------------------------------------------- + core_stdio_fclose - core_file wrapper over fclose +-------------------------------------------------*/ +static int core_stdio_fclose(core_file *file) { + int err = fclose((FILE*)file->argp); + if (err == 0) + free(file); + return err; +} + +/*------------------------------------------------- + core_stdio_fclose_nonowner - don't call fclose because + we don't own the underlying file, but do free the + core_file because libchdr did allocate that itself. +-------------------------------------------------*/ +static int core_stdio_fclose_nonowner(core_file *file) { + free(file); + return 0; +} + +/*------------------------------------------------- + core_stdio_fseek - core_file wrapper over fclose +-------------------------------------------------*/ +static int core_stdio_fseek(core_file* file, int64_t offset, int whence) { + return core_stdio_fseek_impl((FILE*)file->argp, offset, whence); +} diff --git a/libretro-common/formats/libchdr/libchdr_flac.c b/libretro-common/formats/libchdr/libchdr_flac.c index e441c15a49..994bdf72e7 100644 --- a/libretro-common/formats/libchdr/libchdr_flac.c +++ b/libretro-common/formats/libchdr/libchdr_flac.c @@ -8,33 +8,41 @@ ***************************************************************************/ -#include #include +#include + #include -#include -#include +#ifdef HAVE_DR_FLAC +#include +#define DR_FLAC_IMPLEMENTATION +#define DRFLAC_API static INLINE +#include +#endif /*************************************************************************** * FLAC DECODER *************************************************************************** */ -static FLAC__StreamDecoderReadStatus flac_decoder_read_callback_static(const FLAC__StreamDecoder *decoder, FLAC__byte buffer[], size_t *bytes, void *client_data); -FLAC__StreamDecoderReadStatus flac_decoder_read_callback(void* client_data, FLAC__byte buffer[], size_t *bytes); -static void flac_decoder_metadata_callback_static(const FLAC__StreamDecoder *decoder, const FLAC__StreamMetadata *metadata, void *client_data); -static FLAC__StreamDecoderTellStatus flac_decoder_tell_callback_static(const FLAC__StreamDecoder *decoder, FLAC__uint64 *absolute_byte_offset, void *client_data); -static FLAC__StreamDecoderWriteStatus flac_decoder_write_callback_static(const FLAC__StreamDecoder *decoder, const FLAC__Frame *frame, const FLAC__int32 * const buffer[], void *client_data); -FLAC__StreamDecoderWriteStatus flac_decoder_write_callback(void* client_data, const FLAC__Frame *frame, const FLAC__int32 * const buffer[]); -static void flac_decoder_error_callback_static(const FLAC__StreamDecoder *decoder, FLAC__StreamDecoderErrorStatus status, void *client_data); +static size_t flac_decoder_read_callback(void *userdata, void *buffer, size_t bytes); +static drflac_bool32 flac_decoder_seek_callback(void *userdata, int offset, drflac_seek_origin origin); +static void flac_decoder_metadata_callback(void *userdata, drflac_metadata *metadata); +static void flac_decoder_write_callback(void *userdata, void *buffer, size_t bytes); + + +/* getters (valid after reset) */ +/*static uint32_t sample_rate(flac_decoder *decoder) { return decoder->sample_rate; }*/ +static uint8_t channels(flac_decoder *decoder) { return decoder->channels; } +/*static uint8_t bits_per_sample(flac_decoder *decoder) { return decoder->bits_per_sample; }*/ /*------------------------------------------------- * flac_decoder - constructor *------------------------------------------------- */ -void flac_decoder_init(flac_decoder *decoder) +int flac_decoder_init(flac_decoder *decoder) { - decoder->decoder = FLAC__stream_decoder_new(); + decoder->decoder = NULL; decoder->sample_rate = 0; decoder->channels = 0; decoder->bits_per_sample = 0; @@ -46,6 +54,7 @@ void flac_decoder_init(flac_decoder *decoder) decoder->uncompressed_offset = 0; decoder->uncompressed_length = 0; decoder->uncompressed_swap = 0; + return 0; } /*------------------------------------------------- @@ -55,8 +64,10 @@ void flac_decoder_init(flac_decoder *decoder) void flac_decoder_free(flac_decoder* decoder) { - if ((decoder != NULL) && (decoder->decoder != NULL)) - FLAC__stream_decoder_delete(decoder->decoder); + if ((decoder != NULL) && (decoder->decoder != NULL)) { + drflac_close(decoder->decoder); + decoder->decoder = NULL; + } } /*------------------------------------------------- @@ -68,17 +79,11 @@ void flac_decoder_free(flac_decoder* decoder) static int flac_decoder_internal_reset(flac_decoder* decoder) { decoder->compressed_offset = 0; - if (FLAC__stream_decoder_init_stream(decoder->decoder, - &flac_decoder_read_callback_static, - NULL, - &flac_decoder_tell_callback_static, - NULL, - NULL, - &flac_decoder_write_callback_static, - &flac_decoder_metadata_callback_static, - &flac_decoder_error_callback_static, decoder) != FLAC__STREAM_DECODER_INIT_STATUS_OK) - return 0; - return FLAC__stream_decoder_process_until_end_of_metadata(decoder->decoder); + flac_decoder_free(decoder); + decoder->decoder = drflac_open_with_metadata( + flac_decoder_read_callback, flac_decoder_seek_callback, + flac_decoder_metadata_callback, decoder, NULL); + return (decoder->decoder != NULL); } /*------------------------------------------------- @@ -107,16 +112,16 @@ int flac_decoder_reset(flac_decoder* decoder, uint32_t sample_rate, uint8_t num_ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* +2A: start of stream data */ }; memcpy(decoder->custom_header, s_header_template, sizeof(s_header_template)); - decoder->custom_header[0x08] = decoder->custom_header[0x0a] = block_size >> 8; - decoder->custom_header[0x09] = decoder->custom_header[0x0b] = block_size & 0xff; + decoder->custom_header[0x08] = decoder->custom_header[0x0a] = (block_size*num_channels) >> 8; + decoder->custom_header[0x09] = decoder->custom_header[0x0b] = (block_size*num_channels) & 0xff; decoder->custom_header[0x12] = sample_rate >> 12; decoder->custom_header[0x13] = sample_rate >> 4; decoder->custom_header[0x14] = (sample_rate << 4) | ((num_channels - 1) << 1); /* configure the header ahead of the provided buffer */ - decoder->compressed_start = (const FLAC__byte *)(decoder->custom_header); + decoder->compressed_start = (const uint8_t *)(decoder->custom_header); decoder->compressed_length = sizeof(decoder->custom_header); - decoder->compressed2_start = (const FLAC__byte *)(buffer); + decoder->compressed2_start = (const uint8_t *)(buffer); decoder->compressed2_length = length; return flac_decoder_internal_reset(decoder); } @@ -129,6 +134,10 @@ int flac_decoder_reset(flac_decoder* decoder, uint32_t sample_rate, uint8_t num_ int flac_decoder_decode_interleaved(flac_decoder* decoder, int16_t *samples, uint32_t num_samples, int swap_endian) { +#define BUFFER 2352 /* bytes per CD audio sector */ + int16_t buffer[BUFFER]; + uint32_t buf_samples = BUFFER / channels(decoder); + /* configure the uncompressed buffer */ memset(decoder->uncompressed_start, 0, sizeof(decoder->uncompressed_start)); decoder->uncompressed_start[0] = samples; @@ -137,42 +146,16 @@ int flac_decoder_decode_interleaved(flac_decoder* decoder, int16_t *samples, uin decoder->uncompressed_swap = swap_endian; /* loop until we get everything we want */ - while (decoder->uncompressed_offset < decoder->uncompressed_length) - if (!FLAC__stream_decoder_process_single(decoder->decoder)) + while (decoder->uncompressed_offset < decoder->uncompressed_length) { + uint32_t frames = (num_samples < buf_samples ? num_samples : buf_samples); + if (!drflac_read_pcm_frames_s16(decoder->decoder, frames, buffer)) return 0; + flac_decoder_write_callback(decoder, buffer, frames*sizeof(*buffer)*channels(decoder)); + num_samples -= frames; + } return 1; } -#if 0 -/*------------------------------------------------- - * decode - decode to an multiple independent - * data streams - *------------------------------------------------- - */ - -bool flac_decoder::decode(int16_t **samples, uint32_t num_samples, bool swap_endian) -{ - /* make sure we don't have too many channels */ - int chans = channels(); - if (chans > ARRAY_SIZE(m_uncompressed_start)) - return false; - - /* configure the uncompressed buffer */ - memset(m_uncompressed_start, 0, sizeof(m_uncompressed_start)); - for (int curchan = 0; curchan < chans; curchan++) - m_uncompressed_start[curchan] = samples[curchan]; - m_uncompressed_offset = 0; - m_uncompressed_length = num_samples; - m_uncompressed_swap = swap_endian; - - /* loop until we get everything we want */ - while (m_uncompressed_offset < m_uncompressed_length) - if (!FLAC__stream_decoder_process_single(m_decoder)) - return false; - return true; -} -#endif - /*------------------------------------------------- * finish - finish up the decode *------------------------------------------------- @@ -181,16 +164,22 @@ bool flac_decoder::decode(int16_t **samples, uint32_t num_samples, bool swap_end uint32_t flac_decoder_finish(flac_decoder* decoder) { /* get the final decoding position and move forward */ - FLAC__uint64 position = 0; - FLAC__stream_decoder_get_decode_position(decoder->decoder, &position); - FLAC__stream_decoder_finish(decoder->decoder); + drflac *flac = decoder->decoder; + uint64_t position = decoder->compressed_offset; + + /* ugh... there's no function to obtain bytes used in drflac :-/ */ + position -= DRFLAC_CACHE_L2_LINES_REMAINING(&flac->bs) * sizeof(drflac_cache_t); + position -= DRFLAC_CACHE_L1_BITS_REMAINING(&flac->bs) / 8; + position -= flac->bs.unalignedByteCount; /* adjust position if we provided the header */ if (position == 0) return 0; - if (decoder->compressed_start == (const FLAC__byte *)(decoder->custom_header)) + if (decoder->compressed_start == (const uint8_t *)(decoder->custom_header)) position -= decoder->compressed_length; - return (uint32_t)position; + + flac_decoder_free(decoder); + return position; } /*------------------------------------------------- @@ -199,40 +188,31 @@ uint32_t flac_decoder_finish(flac_decoder* decoder) *------------------------------------------------- */ -FLAC__StreamDecoderReadStatus flac_decoder_read_callback_static(const FLAC__StreamDecoder *decoder, FLAC__byte buffer[], size_t *bytes, void *client_data) +static size_t flac_decoder_read_callback(void *userdata, void *buffer, size_t bytes) { - return flac_decoder_read_callback(client_data, buffer, bytes); -} - -FLAC__StreamDecoderReadStatus flac_decoder_read_callback(void* client_data, FLAC__byte buffer[], size_t *bytes) -{ - flac_decoder* decoder = (flac_decoder*)client_data; - - uint32_t expected = (uint32_t)*bytes; + flac_decoder* decoder = (flac_decoder*)userdata; + uint8_t *dst = buffer; /* copy from primary buffer first */ - uint32_t outputpos = 0; - - if (outputpos < *bytes && decoder->compressed_offset < decoder->compressed_length) + uint32_t outputpos = 0; + if (outputpos < bytes && decoder->compressed_offset < decoder->compressed_length) { - uint32_t bytes_to_copy = (uint32_t)MIN(*bytes - outputpos, decoder->compressed_length - decoder->compressed_offset); - memcpy(&buffer[outputpos], decoder->compressed_start + decoder->compressed_offset, bytes_to_copy); + uint32_t bytes_to_copy = MIN(bytes - outputpos, decoder->compressed_length - decoder->compressed_offset); + memcpy(&dst[outputpos], decoder->compressed_start + decoder->compressed_offset, bytes_to_copy); outputpos += bytes_to_copy; decoder->compressed_offset += bytes_to_copy; } /* once we're out of that, copy from the secondary buffer */ - if (outputpos < *bytes && decoder->compressed_offset < decoder->compressed_length + decoder->compressed2_length) + if (outputpos < bytes && decoder->compressed_offset < decoder->compressed_length + decoder->compressed2_length) { - uint32_t bytes_to_copy = (uint32_t)MIN(*bytes - outputpos, decoder->compressed2_length - (decoder->compressed_offset - decoder->compressed_length)); - memcpy(&buffer[outputpos], decoder->compressed2_start + decoder->compressed_offset - decoder->compressed_length, bytes_to_copy); + uint32_t bytes_to_copy = MIN(bytes - outputpos, decoder->compressed2_length - (decoder->compressed_offset - decoder->compressed_length)); + memcpy(&dst[outputpos], decoder->compressed2_start + decoder->compressed_offset - decoder->compressed_length, bytes_to_copy); outputpos += bytes_to_copy; decoder->compressed_offset += bytes_to_copy; } - *bytes = outputpos; - /* return based on whether we ran out of data */ - return (*bytes < expected) ? FLAC__STREAM_DECODER_READ_STATUS_END_OF_STREAM : FLAC__STREAM_DECODER_READ_STATUS_CONTINUE; + return outputpos; } /*------------------------------------------------- @@ -240,30 +220,18 @@ FLAC__StreamDecoderReadStatus flac_decoder_read_callback(void* client_data, FLAC *------------------------------------------------- */ -void flac_decoder_metadata_callback_static(const FLAC__StreamDecoder *decoder, const FLAC__StreamMetadata *metadata, void *client_data) +static void flac_decoder_metadata_callback(void *userdata, drflac_metadata *metadata) { - flac_decoder *fldecoder; + flac_decoder *decoder = userdata; + /* ignore all but STREAMINFO metadata */ - if (metadata->type != FLAC__METADATA_TYPE_STREAMINFO) + if (metadata->type != DRFLAC_METADATA_BLOCK_TYPE_STREAMINFO) return; /* parse out the data we care about */ - fldecoder = (flac_decoder *)(client_data); - fldecoder->sample_rate = metadata->data.stream_info.sample_rate; - fldecoder->bits_per_sample = metadata->data.stream_info.bits_per_sample; - fldecoder->channels = metadata->data.stream_info.channels; -} - -/*------------------------------------------------- - * tell_callback - handle requests to find out - * where in the input stream we are - *------------------------------------------------- - */ - -FLAC__StreamDecoderTellStatus flac_decoder_tell_callback_static(const FLAC__StreamDecoder *decoder, FLAC__uint64 *absolute_byte_offset, void *client_data) -{ - *absolute_byte_offset = ((flac_decoder *)client_data)->compressed_offset; - return FLAC__STREAM_DECODER_TELL_STATUS_OK; + decoder->sample_rate = metadata->data.streaminfo.sampleRate; + decoder->bits_per_sample = metadata->data.streaminfo.bitsPerSample; + decoder->channels = metadata->data.streaminfo.channels; } /*------------------------------------------------- @@ -272,53 +240,66 @@ FLAC__StreamDecoderTellStatus flac_decoder_tell_callback_static(const FLAC__Stre *------------------------------------------------- */ -FLAC__StreamDecoderWriteStatus flac_decoder_write_callback_static(const FLAC__StreamDecoder *decoder, const FLAC__Frame *frame, const FLAC__int32 * const buffer[], void *client_data) -{ - return flac_decoder_write_callback(client_data, frame, buffer); -} - -FLAC__StreamDecoderWriteStatus flac_decoder_write_callback(void *client_data, const FLAC__Frame *frame, const FLAC__int32 * const buffer[]) +static void flac_decoder_write_callback(void *userdata, void *buffer, size_t bytes) { int sampnum, chan; - int shift, blocksize; - flac_decoder * decoder = (flac_decoder *)client_data; - - assert(frame->header.channels == decoder->channels); + int shift, blocksize; + flac_decoder * decoder = (flac_decoder *)userdata; + int16_t *sampbuf = (int16_t *)buffer; + int sampch = channels(decoder); + uint32_t offset = decoder->uncompressed_offset; + uint16_t usample; /* interleaved case */ shift = decoder->uncompressed_swap ? 8 : 0; - blocksize = frame->header.blocksize; + blocksize = bytes / (sampch * sizeof(sampbuf[0])); if (decoder->uncompressed_start[1] == NULL) { - int16_t *dest = decoder->uncompressed_start[0] + decoder->uncompressed_offset * frame->header.channels; - for (sampnum = 0; sampnum < blocksize && decoder->uncompressed_offset < decoder->uncompressed_length; sampnum++, decoder->uncompressed_offset++) - for (chan = 0; chan < (int)frame->header.channels; chan++) - *dest++ = (int16_t)((((uint16_t)buffer[chan][sampnum]) << shift) | (((uint16_t)buffer[chan][sampnum]) >> shift)); + int16_t *dest = decoder->uncompressed_start[0] + offset * sampch; + for (sampnum = 0; sampnum < blocksize && offset < decoder->uncompressed_length; sampnum++, offset++) + for (chan = 0; chan < sampch; chan++) { + usample = (uint16_t)*sampbuf++; + *dest++ = (int16_t)((usample << shift) | (usample >> shift)); + } } /* non-interleaved case */ else { - for (sampnum = 0; sampnum < blocksize && decoder->uncompressed_offset < decoder->uncompressed_length; sampnum++, decoder->uncompressed_offset++) - for (chan = 0; chan < (int)frame->header.channels; chan++) + for (sampnum = 0; sampnum < blocksize && offset < decoder->uncompressed_length; sampnum++, offset++) + for (chan = 0; chan < sampch; chan++) { + usample = (uint16_t)*sampbuf++; if (decoder->uncompressed_start[chan] != NULL) - decoder->uncompressed_start[chan][decoder->uncompressed_offset] = (int16_t) ( (((uint16_t)(buffer[chan][sampnum])) << shift) | ( ((uint16_t)(buffer[chan][sampnum])) >> shift) ); + decoder->uncompressed_start[chan][offset] = (int16_t) ((usample << shift) | (usample >> shift)); + } } - return FLAC__STREAM_DECODER_WRITE_STATUS_CONTINUE; + decoder->uncompressed_offset = offset; } -/** - * @fn void flac_decoder::error_callback_static(const FLAC__StreamDecoder *decoder, FLAC__StreamDecoderErrorStatus status, void *client_data) - * - * @brief ------------------------------------------------- - * error_callback - handle errors (ignore them) - * -------------------------------------------------. - * - * @param decoder The decoder. - * @param status The status. - * @param [in,out] client_data If non-null, information describing the client. + +/*------------------------------------------------- + * seek_callback - handle seeks on the output + * stream + *------------------------------------------------- */ -void flac_decoder_error_callback_static(const FLAC__StreamDecoder *decoder, FLAC__StreamDecoderErrorStatus status, void *client_data) +static drflac_bool32 flac_decoder_seek_callback(void *userdata, int offset, drflac_seek_origin origin) { + flac_decoder * decoder = (flac_decoder *)userdata; + uint32_t length = decoder->compressed_length + decoder->compressed2_length; + + if (origin == drflac_seek_origin_start) { + uint32_t pos = offset; + if (pos <= length) { + decoder->compressed_offset = pos; + return 1; + } + } else if (origin == drflac_seek_origin_current) { + uint32_t pos = decoder->compressed_offset + offset; + if (pos <= length) { + decoder->compressed_offset = pos; + return 1; + } + } + return 0; } diff --git a/libretro-common/formats/libchdr/libchdr_flac_codec.c b/libretro-common/formats/libchdr/libchdr_flac_codec.c index 3a6e669268..463e2c6ac4 100644 --- a/libretro-common/formats/libchdr/libchdr_flac_codec.c +++ b/libretro-common/formats/libchdr/libchdr_flac_codec.c @@ -47,40 +47,94 @@ #include #include #include +#include #include #include #include -#define TRUE 1 -#define FALSE 0 - /*************************************************************************** * CD FLAC DECOMPRESSOR *************************************************************************** */ /*------------------------------------------------------ - * cdfl_codec_blocksize - return the optimal block size + * flac_codec_blocksize - return the optimal block size *------------------------------------------------------ */ -static uint32_t cdfl_codec_blocksize(uint32_t bytes) +static uint32_t flac_codec_blocksize(uint32_t bytes) { /* determine FLAC block size, which must be 16-65535 * clamp to 2k since that's supposed to be the sweet spot */ - uint32_t hunkbytes = bytes / 4; - while (hunkbytes > 2048) - hunkbytes /= 2; - return hunkbytes; + uint32_t blocksize = bytes / 4; + while (blocksize > 2048) + blocksize /= 2; + return blocksize; +} + +chd_error flac_codec_init(void *codec, uint32_t hunkbytes) +{ + uint16_t native_endian = 0; + flac_codec_data *flac = (flac_codec_data*)codec; + + /* make sure the CHD's hunk size is an even multiple of the sample size */ + if (hunkbytes % 4 != 0) + return CHDERR_CODEC_ERROR; + + /* determine whether we want native or swapped samples */ + *(uint8_t *)(&native_endian) = 1; + flac->native_endian = (native_endian & 1); + + /* flac decoder init */ + if (flac_decoder_init(&flac->decoder)) + return CHDERR_OUT_OF_MEMORY; + + return CHDERR_NONE; +} + +void flac_codec_free(void *codec) +{ + flac_codec_data *flac = (flac_codec_data*)codec; + flac_decoder_free(&flac->decoder); +} + +chd_error flac_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + flac_codec_data *flac = (flac_codec_data*)codec; + int swap_endian; + + if (src[0] == 'L') + swap_endian = !flac->native_endian; + else if (src[0] == 'B') + swap_endian = flac->native_endian; + else + return CHDERR_DECOMPRESSION_ERROR; + + if (!flac_decoder_reset(&flac->decoder, 44100, 2, flac_codec_blocksize(destlen), src + 1, complen - 1)) + return CHDERR_DECOMPRESSION_ERROR; + if (!flac_decoder_decode_interleaved(&flac->decoder, (int16_t *)(dest), destlen/4, swap_endian)) + return CHDERR_DECOMPRESSION_ERROR; + flac_decoder_finish(&flac->decoder); + + return CHDERR_NONE; +} + +static uint32_t cdfl_codec_blocksize(uint32_t bytes) +{ + /* for CDs it seems that CD_MAX_SECTOR_DATA is the right target */ + uint32_t blocksize = bytes / 4; + while (blocksize > CD_MAX_SECTOR_DATA) + blocksize /= 2; + return blocksize; } chd_error cdfl_codec_init(void *codec, uint32_t hunkbytes) { #ifdef WANT_SUBCODE - chd_error ret; + chd_error ret; #endif - uint16_t native_endian = 0; + uint16_t native_endian = 0; cdfl_codec_data *cdfl = (cdfl_codec_data*)codec; /* make sure the CHD's hunk size is an even multiple of the frame size */ @@ -103,8 +157,7 @@ chd_error cdfl_codec_init(void *codec, uint32_t hunkbytes) #endif /* flac decoder init */ - flac_decoder_init(&cdfl->decoder); - if (cdfl->decoder.decoder == NULL) + if (flac_decoder_init(&cdfl->decoder)) return CHDERR_OUT_OF_MEMORY; return CHDERR_NONE; @@ -124,10 +177,10 @@ void cdfl_codec_free(void *codec) chd_error cdfl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) { uint32_t framenum; - uint8_t *buffer; + uint8_t *buffer; #ifdef WANT_SUBCODE - uint32_t offset; - chd_error ret; + uint32_t offset; + chd_error ret; #endif cdfl_codec_data *cdfl = (cdfl_codec_data*)codec; diff --git a/libretro-common/formats/libchdr/libchdr_huffman.c b/libretro-common/formats/libchdr/libchdr_huffman.c index 8f1f095ba0..380e0f2add 100644 --- a/libretro-common/formats/libchdr/libchdr_huffman.c +++ b/libretro-common/formats/libchdr/libchdr_huffman.c @@ -97,7 +97,6 @@ ***************************************************************************/ #include -#include #include #include @@ -124,7 +123,8 @@ struct huffman_decoder* create_huffman_decoder(int numcodes, int maxbits) { - struct huffman_decoder* decoder; + struct huffman_decoder* decoder = NULL; + /* limit to 24 bits */ if (maxbits > 24) return NULL; @@ -179,10 +179,11 @@ uint32_t huffman_decode_one(struct huffman_decoder* decoder, struct bitstream* b enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, struct bitstream* bitbuf) { - enum huffman_error error; - /* bits per entry depends on the maxbits */ int numbits; - int curnode; + uint32_t curnode; + enum huffman_error error; + + /* bits per entry depends on the maxbits */ if (decoder->maxbits >= 16) numbits = 5; else if (decoder->maxbits >= 8) @@ -191,7 +192,7 @@ enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, stru numbits = 3; /* loop until we read all the nodes */ - for (curnode = 0; curnode < (int)decoder->numcodes; /* blank */) + for (curnode = 0; curnode < decoder->numcodes; ) { /* a non-one value is just raw */ int nodebits = bitstream_read(bitbuf, numbits); @@ -210,6 +211,8 @@ enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, stru else { int repcount = bitstream_read(bitbuf, numbits) + 3; + if (repcount + curnode > decoder->numcodes) + return HUFFERR_INVALID_DATA; while (repcount--) decoder->huffnode[curnode++].numbits = nodebits; } @@ -217,7 +220,7 @@ enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, stru } /* make sure we ended up with the right number */ - if (curnode != (int)decoder->numcodes) + if (curnode != decoder->numcodes) return HUFFERR_INVALID_DATA; /* assign canonical codes for all nodes based on their code lengths */ @@ -232,6 +235,7 @@ enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, stru return bitstream_overflow(bitbuf) ? HUFFERR_INPUT_BUFFER_TOO_SMALL : HUFFERR_NONE; } + /*------------------------------------------------- * import_tree_huffman - import a huffman-encoded * huffman tree from a source data stream @@ -240,19 +244,18 @@ enum huffman_error huffman_import_tree_rle(struct huffman_decoder* decoder, stru enum huffman_error huffman_import_tree_huffman(struct huffman_decoder* decoder, struct bitstream* bitbuf) { + int start; int last = 0; - int curcode; - uint32_t temp; - enum huffman_error error; + int count = 0; + int index; + uint32_t curcode; uint8_t rlefullbits = 0; - int index, count = 0; - int start; + uint32_t temp; + enum huffman_error error; /* start by parsing the lengths for the small tree */ struct huffman_decoder* smallhuff = create_huffman_decoder(24, 6); - smallhuff->huffnode[0].numbits = bitstream_read(bitbuf, 3); start = bitstream_read(bitbuf, 3) + 1; - for (index = 1; index < 24; index++) { if (index < start || count == 7) @@ -273,13 +276,10 @@ enum huffman_error huffman_import_tree_huffman(struct huffman_decoder* decoder, /* determine the maximum length of an RLE count */ temp = decoder->numcodes - 9; while (temp != 0) - { - temp >>= 1; - rlefullbits++; - } + temp >>= 1, rlefullbits++; /* now process the rest of the data */ - for (curcode = 0; curcode < (int)decoder->numcodes; /* blank */) + for (curcode = 0; curcode < decoder->numcodes; ) { int value = huffman_decode_one(smallhuff, bitbuf); if (value != 0) @@ -289,29 +289,25 @@ enum huffman_error huffman_import_tree_huffman(struct huffman_decoder* decoder, int count = bitstream_read(bitbuf, 3) + 2; if (count == 7+2) count += bitstream_read(bitbuf, rlefullbits); - for (/* blank */; count != 0 && curcode < (int)decoder->numcodes; count--) + for ( ; count != 0 && curcode < decoder->numcodes; count--) decoder->huffnode[curcode++].numbits = last; } } + /* make sure we free the local huffman decoder */ + delete_huffman_decoder(smallhuff); + /* make sure we ended up with the right number */ - if (curcode != (int)decoder->numcodes) - { - delete_huffman_decoder(smallhuff); + if (curcode != decoder->numcodes) return HUFFERR_INVALID_DATA; - } /* assign canonical codes for all nodes based on their code lengths */ error = huffman_assign_canonical_codes(decoder); if (error != HUFFERR_NONE) - { - delete_huffman_decoder(smallhuff); return error; - } /* build the lookup table */ huffman_build_lookup_table(decoder); - delete_huffman_decoder(smallhuff); /* determine final input length and report errors */ return bitstream_overflow(bitbuf) ? HUFFERR_INPUT_BUFFER_TOO_SMALL : HUFFERR_NONE; @@ -325,18 +321,18 @@ enum huffman_error huffman_import_tree_huffman(struct huffman_decoder* decoder, enum huffman_error huffman_compute_tree_from_histo(struct huffman_decoder* decoder) { + uint32_t i; + uint32_t lowerweight; + uint32_t upperweight; /* compute the number of data items in the histogram */ - int i; - uint32_t upperweight; - uint32_t lowerweight = 0; uint32_t sdatacount = 0; - for (i = 0; i < (int)decoder->numcodes; i++) + for (i = 0; i < decoder->numcodes; i++) sdatacount += decoder->datahisto[i]; /* binary search to achieve the optimum encoding */ + lowerweight = 0; upperweight = sdatacount * 2; - - for (;;) + while (1) { /* build a tree using the current weight */ uint32_t curweight = (upperweight + lowerweight) / 2; @@ -389,17 +385,14 @@ static int huffman_tree_node_compare(const void *item1, const void *item2) int huffman_build_tree(struct huffman_decoder* decoder, uint32_t totaldata, uint32_t totalweight) { - int nextalloc; + uint32_t curcode; + int nextalloc; + int listitems = 0; int maxbits = 0; /* make a list of all non-zero nodes */ - struct node_t** list = (struct node_t**) - malloc(sizeof(struct node_t*) * decoder->numcodes * 2); - int curcode, listitems = 0; - - memset(decoder->huffnode, 0, - decoder->numcodes * sizeof(decoder->huffnode[0])); - - for (curcode = 0; curcode < (int)decoder->numcodes; curcode++) + struct node_t** list = (struct node_t**)malloc(sizeof(struct node_t*) * decoder->numcodes * 2); + memset(decoder->huffnode, 0, decoder->numcodes * sizeof(decoder->huffnode[0])); + for (curcode = 0; curcode < decoder->numcodes; curcode++) if (decoder->datahisto[curcode] != 0) { list[listitems++] = &decoder->huffnode[curcode]; @@ -407,65 +400,57 @@ int huffman_build_tree(struct huffman_decoder* decoder, uint32_t totaldata, uint decoder->huffnode[curcode].bits = curcode; /* scale the weight by the current effective length, ensuring we don't go to 0 */ - decoder->huffnode[curcode].weight = (uint32_t)(((uint64_t)decoder->datahisto[curcode]) * ((uint64_t)totalweight) / ((uint64_t)totaldata)); + decoder->huffnode[curcode].weight = ((uint64_t)decoder->datahisto[curcode]) * ((uint64_t)totalweight) / ((uint64_t)totaldata); if (decoder->huffnode[curcode].weight == 0) decoder->huffnode[curcode].weight = 1; } #if 0 - { - unsigned i; - fprintf(stderr, "Pre-sort:\n"); - for (i = 0; i < listitems; i++) - fprintf(stderr, "weight: %d code: %d\n", - list[i]->m_weight, list[i]->m_bits); - } + fprintf(stderr, "Pre-sort:\n"); + for (int i = 0; i < listitems; i++) { + fprintf(stderr, "weight: %d code: %d\n", list[i]->m_weight, list[i]->m_bits); + } #endif /* sort the list by weight, largest weight first */ qsort(&list[0], listitems, sizeof(list[0]), huffman_tree_node_compare); #if 0 - fprintf(stderr, "Post-sort:\n"); - for (int i = 0; i < listitems; i++) - fprintf(stderr, "weight: %d code: %d\n", list[i]->m_weight, list[i]->m_bits); - fprintf(stderr, "===================\n"); + fprintf(stderr, "Post-sort:\n"); + for (int i = 0; i < listitems; i++) { + fprintf(stderr, "weight: %d code: %d\n", list[i]->m_weight, list[i]->m_bits); + } + fprintf(stderr, "===================\n"); #endif /* now build the tree */ nextalloc = decoder->numcodes; - while (listitems > 1) { int curitem; /* remove lowest two items */ - struct node_t* node1 = &(*list[--listitems]); - struct node_t* node0 = &(*list[--listitems]); + struct node_t* node1 = &(*list[--listitems]); + struct node_t* node0 = &(*list[--listitems]); /* create new node */ struct node_t* newnode = &decoder->huffnode[nextalloc++]; - newnode->parent = NULL; - node0->parent = - node1->parent = newnode; - newnode->weight = - node0->weight + node1->weight; + newnode->parent = NULL; + node0->parent = node1->parent = newnode; + newnode->weight = node0->weight + node1->weight; /* insert into list at appropriate location */ for (curitem = 0; curitem < listitems; curitem++) if (newnode->weight > list[curitem]->weight) { - memmove(&list[curitem+1], - &list[curitem], - (listitems - curitem) * sizeof(list[0])); + memmove(&list[curitem+1], &list[curitem], (listitems - curitem) * sizeof(list[0])); break; } list[curitem] = newnode; listitems++; } - /* compute the number of bits in each code, - * and fill in another histogram */ - for (curcode = 0; curcode < (int)decoder->numcodes; curcode++) + /* compute the number of bits in each code, and fill in another histogram */ + for (curcode = 0; curcode < decoder->numcodes; curcode++) { struct node_t *curnode; struct node_t* node = &decoder->huffnode[curcode]; @@ -476,8 +461,7 @@ int huffman_build_tree(struct huffman_decoder* decoder, uint32_t totaldata, uint if (node->weight > 0) { /* determine the number of bits for this node */ - for (curnode = node; - curnode->parent != NULL; curnode = curnode->parent) + for (curnode = node; curnode->parent != NULL; curnode = curnode->parent) node->numbits++; if (node->numbits == 0) node->numbits = 1; @@ -486,7 +470,6 @@ int huffman_build_tree(struct huffman_decoder* decoder, uint32_t totaldata, uint maxbits = MAX(maxbits, ((int)node->numbits)); } } - free(list); return maxbits; } @@ -499,11 +482,12 @@ int huffman_build_tree(struct huffman_decoder* decoder, uint32_t totaldata, uint enum huffman_error huffman_assign_canonical_codes(struct huffman_decoder* decoder) { + uint32_t curcode; + int codelen; uint32_t curstart = 0; /* build up a histogram of bit lengths */ - int curcode, codelen; uint32_t bithisto[33] = { 0 }; - for (curcode = 0; curcode < (int)decoder->numcodes; curcode++) + for (curcode = 0; curcode < decoder->numcodes; curcode++) { struct node_t* node = &decoder->huffnode[curcode]; if (node->numbits > decoder->maxbits) @@ -523,7 +507,7 @@ enum huffman_error huffman_assign_canonical_codes(struct huffman_decoder* decode } /* now assign canonical codes */ - for (curcode = 0; curcode < (int)decoder->numcodes; curcode++) + for (curcode = 0; curcode < decoder->numcodes; curcode++) { struct node_t* node = &decoder->huffnode[curcode]; if (node->numbits > 0) @@ -540,21 +524,24 @@ enum huffman_error huffman_assign_canonical_codes(struct huffman_decoder* decode void huffman_build_lookup_table(struct huffman_decoder* decoder) { + uint32_t curcode; /* iterate over all codes */ - int curcode; - for (curcode = 0; curcode < (int)decoder->numcodes; curcode++) + for (curcode = 0; curcode < decoder->numcodes; curcode++) { /* process all nodes which have non-zero bits */ struct node_t* node = &decoder->huffnode[curcode]; if (node->numbits > 0) { + int shift; + lookup_value *dest; + lookup_value *destend; /* set up the entry */ lookup_value value = MAKE_LOOKUP(curcode, node->numbits); /* fill all matching entries */ - int shift = decoder->maxbits - node->numbits; - lookup_value *dest = &decoder->lookup[node->bits << shift]; - lookup_value *destend = &decoder->lookup[((node->bits + 1) << shift) - 1]; + shift = decoder->maxbits - node->numbits; + dest = &decoder->lookup[node->bits << shift]; + destend = &decoder->lookup[((node->bits + 1) << shift) - 1]; while (dest <= destend) *dest++ = value; } diff --git a/libretro-common/formats/libchdr/libchdr_lzma.c b/libretro-common/formats/libchdr/libchdr_lzma.c index f4a009d18d..8a1d7489fc 100644 --- a/libretro-common/formats/libchdr/libchdr_lzma.c +++ b/libretro-common/formats/libchdr/libchdr_lzma.c @@ -52,14 +52,48 @@ #include #include -#define TRUE 1 -#define FALSE 0 - /*************************************************************************** * LZMA ALLOCATOR HELPER *************************************************************************** */ +void *lzma_fast_alloc(void *p, size_t size); +void lzma_fast_free(void *p, void *address); + +/*------------------------------------------------- + * lzma_allocator_init + *------------------------------------------------- + */ + +static void lzma_allocator_init(void* p) +{ + lzma_allocator *codec = (lzma_allocator *)(p); + + /* reset pointer list */ + memset(codec->allocptr, 0, sizeof(codec->allocptr)); + memset(codec->allocptr2, 0, sizeof(codec->allocptr2)); + codec->Alloc = lzma_fast_alloc; + codec->Free = lzma_fast_free; +} + +/*------------------------------------------------- + * lzma_allocator_free + *------------------------------------------------- + */ + +static void lzma_allocator_free(void* p ) +{ + int i; + lzma_allocator *codec = (lzma_allocator *)(p); + + /* free our memory */ + for (i = 0 ; i < MAX_LZMA_ALLOCS ; i++) + { + if (codec->allocptr[i] != NULL) + free(codec->allocptr[i]); + } +} + /*------------------------------------------------- * lzma_fast_alloc - fast malloc for lzma, which * allocates and frees memory frequently @@ -70,7 +104,7 @@ #define LZMA_MIN_ALIGNMENT_BITS 512 #define LZMA_MIN_ALIGNMENT_BYTES (LZMA_MIN_ALIGNMENT_BITS / 8) -static void *lzma_fast_alloc(void *p, size_t size) +void *lzma_fast_alloc(void *p, size_t size) { int scan; uint32_t *addr = NULL; @@ -114,7 +148,7 @@ static void *lzma_fast_alloc(void *p, size_t size) } /* set the low bit of the size so we don't match next time */ - *addr = (uint32_t)size | 1; + *addr = size | 1; /* return aligned address */ return (void*)vaddr; @@ -125,7 +159,8 @@ static void *lzma_fast_alloc(void *p, size_t size) * allocates and frees memory frequently *------------------------------------------------- */ -static void lzma_fast_free(void *p, void *address) + +void lzma_fast_free(void *p, void *address) { int scan; uint32_t *ptr = NULL; @@ -149,39 +184,6 @@ static void lzma_fast_free(void *p, void *address) } } -/*------------------------------------------------- - * lzma_allocator_init - *------------------------------------------------- - */ - -void lzma_allocator_init(void* p) -{ - lzma_allocator *codec = (lzma_allocator *)(p); - - /* reset pointer list */ - memset(codec->allocptr, 0, sizeof(codec->allocptr)); - codec->Alloc = lzma_fast_alloc; - codec->Free = lzma_fast_free; -} - -/*------------------------------------------------- - * lzma_allocator_free - *------------------------------------------------- - */ - -void lzma_allocator_free(void* p ) -{ - lzma_allocator *codec = (lzma_allocator *)(p); - - /* free our memory */ - int i; - for (i = 0 ; i < MAX_LZMA_ALLOCS ; i++) - { - if (codec->allocptr[i] != NULL) - free(codec->allocptr[i]); - } -} - /*************************************************************************** * LZMA DECOMPRESSOR *************************************************************************** @@ -194,11 +196,11 @@ void lzma_allocator_free(void* p ) chd_error lzma_codec_init(void* codec, uint32_t hunkbytes) { + CLzmaEncHandle enc; CLzmaEncProps encoder_props; - CLzmaEncHandle enc; - uint8_t decoder_props[LZMA_PROPS_SIZE]; - lzma_allocator* alloc; - size_t props_size; + Byte decoder_props[LZMA_PROPS_SIZE]; + SizeT props_size; + lzma_allocator* alloc; lzma_codec_data* lzma_codec = (lzma_codec_data*) codec; /* construct the decoder */ @@ -250,11 +252,10 @@ chd_error lzma_codec_init(void* codec, uint32_t hunkbytes) void lzma_codec_free(void* codec) { lzma_codec_data* lzma_codec = (lzma_codec_data*) codec; - lzma_allocator* alloc = &lzma_codec->allocator; /* free memory */ LzmaDec_Free(&lzma_codec->decoder, (ISzAlloc*)&lzma_codec->allocator); - lzma_allocator_free(alloc); + lzma_allocator_free(&lzma_codec->allocator); } /*------------------------------------------------- @@ -266,8 +267,8 @@ void lzma_codec_free(void* codec) chd_error lzma_codec_decompress(void* codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) { ELzmaStatus status; - SRes res; - size_t consumedlen, decodedlen; + SRes res; + SizeT consumedlen, decodedlen; /* initialize */ lzma_codec_data* lzma_codec = (lzma_codec_data*) codec; LzmaDec_Init(&lzma_codec->decoder); @@ -292,36 +293,35 @@ chd_error cdlz_codec_init(void* codec, uint32_t hunkbytes) if (cdlz->buffer == NULL) return CHDERR_OUT_OF_MEMORY; + /* make sure the CHD's hunk size is an even multiple of the frame size */ ret = lzma_codec_init(&cdlz->base_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); if (ret != CHDERR_NONE) return ret; #ifdef WANT_SUBCODE - ret = zlib_codec_init(&cdlz->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); + ret = zlib_codec_init(&cdlz->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SUBCODE_DATA); if (ret != CHDERR_NONE) return ret; #endif + if (hunkbytes % CD_FRAME_SIZE != 0) + return CHDERR_CODEC_ERROR; + return CHDERR_NONE; } void cdlz_codec_free(void* codec) { cdlz_codec_data* cdlz = (cdlz_codec_data*) codec; - + free(cdlz->buffer); lzma_codec_free(&cdlz->base_decompressor); #ifdef WANT_SUBCODE zlib_codec_free(&cdlz->subcode_decompressor); #endif - if (cdlz->buffer) - free(cdlz->buffer); } chd_error cdlz_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) { -#ifdef WANT_RAW_DATA_SECTOR - uint8_t *sector; -#endif uint32_t framenum; cdlz_codec_data* cdlz = (cdlz_codec_data*)codec; @@ -339,14 +339,14 @@ chd_error cdlz_codec_decompress(void *codec, const uint8_t *src, uint32_t comple /* reset and decode */ lzma_codec_decompress(&cdlz->base_decompressor, &src[header_bytes], complen_base, &cdlz->buffer[0], frames * CD_MAX_SECTOR_DATA); #ifdef WANT_SUBCODE - if (header_bytes + complen_base >= complen) - return CHDERR_DECOMPRESSION_ERROR; zlib_codec_decompress(&cdlz->subcode_decompressor, &src[header_bytes + complen_base], complen - complen_base - header_bytes, &cdlz->buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); #endif /* reassemble the data */ for (framenum = 0; framenum < frames; framenum++) { + uint8_t *sector; + memcpy(&dest[framenum * CD_FRAME_SIZE], &cdlz->buffer[framenum * CD_MAX_SECTOR_DATA], CD_MAX_SECTOR_DATA); #ifdef WANT_SUBCODE memcpy(&dest[framenum * CD_FRAME_SIZE + CD_MAX_SECTOR_DATA], &cdlz->buffer[frames * CD_MAX_SECTOR_DATA + framenum * CD_MAX_SUBCODE_DATA], CD_MAX_SUBCODE_DATA); diff --git a/libretro-common/formats/libchdr/libchdr_zlib.c b/libretro-common/formats/libchdr/libchdr_zlib.c index e02ab31722..79b183746b 100644 --- a/libretro-common/formats/libchdr/libchdr_zlib.c +++ b/libretro-common/formats/libchdr/libchdr_zlib.c @@ -52,95 +52,6 @@ #include #include -#define TRUE 1 -#define FALSE 0 - -/* cdzl */ - -chd_error cdzl_codec_init(void *codec, uint32_t hunkbytes) -{ - chd_error ret; - cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; - - /* make sure the CHD's hunk size is an even multiple of the frame size */ - if (hunkbytes % CD_FRAME_SIZE != 0) - return CHDERR_CODEC_ERROR; - - cdzl->buffer = (uint8_t*)malloc(sizeof(uint8_t) * hunkbytes); - if (cdzl->buffer == NULL) - return CHDERR_OUT_OF_MEMORY; - - ret = zlib_codec_init(&cdzl->base_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); - if (ret != CHDERR_NONE) - return ret; - -#ifdef WANT_SUBCODE - ret = zlib_codec_init(&cdzl->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); - if (ret != CHDERR_NONE) - return ret; -#endif - - return CHDERR_NONE; -} - -void cdzl_codec_free(void *codec) -{ - cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; - - zlib_codec_free(&cdzl->base_decompressor); -#ifdef WANT_SUBCODE - zlib_codec_free(&cdzl->subcode_decompressor); -#endif - if (cdzl->buffer) - free(cdzl->buffer); -} - -chd_error cdzl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) -{ -#ifdef WANT_RAW_DATA_SECTOR - uint8_t *sector; -#endif - uint32_t framenum; - cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; - - /* determine header bytes */ - uint32_t frames = destlen / CD_FRAME_SIZE; - uint32_t complen_bytes = (destlen < 65536) ? 2 : 3; - uint32_t ecc_bytes = (frames + 7) / 8; - uint32_t header_bytes = ecc_bytes + complen_bytes; - - /* extract compressed length of base */ - uint32_t complen_base = (src[ecc_bytes + 0] << 8) | src[ecc_bytes + 1]; - if (complen_bytes > 2) - complen_base = (complen_base << 8) | src[ecc_bytes + 2]; - - /* reset and decode */ - zlib_codec_decompress(&cdzl->base_decompressor, &src[header_bytes], complen_base, &cdzl->buffer[0], frames * CD_MAX_SECTOR_DATA); -#ifdef WANT_SUBCODE - zlib_codec_decompress(&cdzl->subcode_decompressor, &src[header_bytes + complen_base], complen - complen_base - header_bytes, &cdzl->buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); -#endif - - /* reassemble the data */ - for (framenum = 0; framenum < frames; framenum++) - { - memcpy(&dest[framenum * CD_FRAME_SIZE], &cdzl->buffer[framenum * CD_MAX_SECTOR_DATA], CD_MAX_SECTOR_DATA); -#ifdef WANT_SUBCODE - memcpy(&dest[framenum * CD_FRAME_SIZE + CD_MAX_SECTOR_DATA], &cdzl->buffer[frames * CD_MAX_SECTOR_DATA + framenum * CD_MAX_SUBCODE_DATA], CD_MAX_SUBCODE_DATA); -#endif - -#ifdef WANT_RAW_DATA_SECTOR - /* reconstitute the ECC data and sync header */ - sector = (uint8_t *)&dest[framenum * CD_FRAME_SIZE]; - if ((src[framenum / 8] & (1 << (framenum % 8))) != 0) - { - memcpy(sector, s_cd_sync_header, sizeof(s_cd_sync_header)); - ecc_generate(sector); - } -#endif - } - return CHDERR_NONE; -} - /*************************************************************************** ZLIB COMPRESSION CODEC ***************************************************************************/ @@ -174,10 +85,6 @@ chd_error zlib_codec_init(void *codec, uint32_t hunkbytes) else err = CHDERR_NONE; - /* handle an error */ - if (err != CHDERR_NONE) - zlib_codec_free(data); - return err; } @@ -193,21 +100,15 @@ void zlib_codec_free(void *codec) /* deinit the streams */ if (data != NULL) { - int i; - zlib_allocator alloc; - inflateEnd(&data->inflater); /* free our fast memory */ - alloc = data->allocator; - for (i = 0; i < MAX_ZLIB_ALLOCS; i++) - if (alloc.allocptr[i]) - free(alloc.allocptr[i]); + zlib_allocator_free(&data->allocator); } } /*------------------------------------------------- - zlib_codec_decompress - decomrpess data using + zlib_codec_decompress - decompress data using the ZLIB codec -------------------------------------------------*/ @@ -229,7 +130,6 @@ chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t comple /* do it */ zerr = inflate(&data->inflater, Z_FINISH); - (void)zerr; if (data->inflater.total_out != destlen) return CHDERR_DECOMPRESSION_ERROR; @@ -249,7 +149,7 @@ voidpf zlib_fast_alloc(voidpf opaque, uInt items, uInt size) { zlib_allocator *alloc = (zlib_allocator *)opaque; uintptr_t paddr = 0; - UINT32 *ptr; + uint32_t *ptr; int i; /* compute the size, rounding to the nearest 1k */ @@ -270,7 +170,7 @@ voidpf zlib_fast_alloc(voidpf opaque, uInt items, uInt size) } /* alloc a new one */ - ptr = (UINT32 *)malloc(size + sizeof(UINT32) + ZLIB_MIN_ALIGNMENT_BYTES); + ptr = (uint32_t *)malloc(size + sizeof(uint32_t) + ZLIB_MIN_ALIGNMENT_BYTES); if (!ptr) return NULL; @@ -279,7 +179,7 @@ voidpf zlib_fast_alloc(voidpf opaque, uInt items, uInt size) if (!alloc->allocptr[i]) { alloc->allocptr[i] = ptr; - paddr = (((uintptr_t)ptr) + sizeof(UINT32) + (ZLIB_MIN_ALIGNMENT_BYTES-1)) & (~(ZLIB_MIN_ALIGNMENT_BYTES-1)); + paddr = (((uintptr_t)ptr) + sizeof(uint32_t) + (ZLIB_MIN_ALIGNMENT_BYTES-1)) & (~(ZLIB_MIN_ALIGNMENT_BYTES-1)); alloc->allocptr2[i] = (uint32_t*)paddr; break; } @@ -299,7 +199,7 @@ voidpf zlib_fast_alloc(voidpf opaque, uInt items, uInt size) void zlib_fast_free(voidpf opaque, voidpf address) { zlib_allocator *alloc = (zlib_allocator *)opaque; - UINT32 *ptr = (UINT32 *)address; + uint32_t *ptr = (uint32_t *)address; int i; /* find the hunk */ @@ -311,3 +211,101 @@ void zlib_fast_free(voidpf opaque, voidpf address) return; } } + +/*------------------------------------------------- + zlib_allocator_free +-------------------------------------------------*/ +void zlib_allocator_free(voidpf opaque) +{ + zlib_allocator *alloc = (zlib_allocator *)opaque; + int i; + + for (i = 0; i < MAX_ZLIB_ALLOCS; i++) + if (alloc->allocptr[i]) + free(alloc->allocptr[i]); +} + + +/* cdzl */ + +chd_error cdzl_codec_init(void *codec, uint32_t hunkbytes) +{ + chd_error ret; + cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; + + /* make sure the CHD's hunk size is an even multiple of the frame size */ + if (hunkbytes % CD_FRAME_SIZE != 0) + return CHDERR_CODEC_ERROR; + + cdzl->buffer = (uint8_t*)malloc(sizeof(uint8_t) * hunkbytes); + if (cdzl->buffer == NULL) + return CHDERR_OUT_OF_MEMORY; + + ret = zlib_codec_init(&cdzl->base_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); + if (ret != CHDERR_NONE) + return ret; + +#ifdef WANT_SUBCODE + ret = zlib_codec_init(&cdzl->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SUBCODE_DATA); + if (ret != CHDERR_NONE) + return ret; +#endif + + return CHDERR_NONE; +} + +void cdzl_codec_free(void *codec) +{ + cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; + zlib_codec_free(&cdzl->base_decompressor); +#ifdef WANT_SUBCODE + zlib_codec_free(&cdzl->subcode_decompressor); +#endif + free(cdzl->buffer); +} + +chd_error cdzl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + uint32_t framenum; + cdzl_codec_data* cdzl = (cdzl_codec_data*)codec; + + /* determine header bytes */ + uint32_t frames = destlen / CD_FRAME_SIZE; + uint32_t complen_bytes = (destlen < 65536) ? 2 : 3; + uint32_t ecc_bytes = (frames + 7) / 8; + uint32_t header_bytes = ecc_bytes + complen_bytes; + + /* extract compressed length of base */ + uint32_t complen_base = (src[ecc_bytes + 0] << 8) | src[ecc_bytes + 1]; + if (complen_bytes > 2) + complen_base = (complen_base << 8) | src[ecc_bytes + 2]; + + /* reset and decode */ + zlib_codec_decompress(&cdzl->base_decompressor, &src[header_bytes], complen_base, &cdzl->buffer[0], frames * CD_MAX_SECTOR_DATA); +#ifdef WANT_SUBCODE + zlib_codec_decompress(&cdzl->subcode_decompressor, &src[header_bytes + complen_base], complen - complen_base - header_bytes, &cdzl->buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); +#endif + + /* reassemble the data */ + for (framenum = 0; framenum < frames; framenum++) + { + uint8_t *sector; + + memcpy(&dest[framenum * CD_FRAME_SIZE], &cdzl->buffer[framenum * CD_MAX_SECTOR_DATA], CD_MAX_SECTOR_DATA); +#ifdef WANT_SUBCODE + memcpy(&dest[framenum * CD_FRAME_SIZE + CD_MAX_SECTOR_DATA], &cdzl->buffer[frames * CD_MAX_SECTOR_DATA + framenum * CD_MAX_SUBCODE_DATA], CD_MAX_SUBCODE_DATA); +#endif + +#ifdef WANT_RAW_DATA_SECTOR + /* reconstitute the ECC data and sync header */ + sector = (uint8_t *)&dest[framenum * CD_FRAME_SIZE]; + if ((src[framenum / 8] & (1 << (framenum % 8))) != 0) + { + memcpy(sector, s_cd_sync_header, sizeof(s_cd_sync_header)); + ecc_generate(sector); + } +#endif + } + return CHDERR_NONE; +} + diff --git a/libretro-common/formats/libchdr/libchdr_zstd.c b/libretro-common/formats/libchdr/libchdr_zstd.c new file mode 100644 index 0000000000..001887b650 --- /dev/null +++ b/libretro-common/formats/libchdr/libchdr_zstd.c @@ -0,0 +1,220 @@ +/*************************************************************************** + + libchdr_zstd.c + + MAME Compressed Hunks of Data file format + +**************************************************************************** + + Copyright Aaron Giles + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name 'MAME' nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY AARON GILES ''AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL AARON GILES BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/*************************************************************************** + * ZSTD DECOMPRESSOR + *************************************************************************** + */ + +/*------------------------------------------------- + * zstd_codec_init - constructor + *------------------------------------------------- + */ + +chd_error zstd_codec_init(void* codec, uint32_t hunkbytes) +{ + zstd_codec_data* zstd_codec = (zstd_codec_data*) codec; + + zstd_codec->dstream = ZSTD_createDStream(); + if (!zstd_codec->dstream) { + printf("NO DSTREAM CREATED!\n"); + return CHDERR_DECOMPRESSION_ERROR; + } + return CHDERR_NONE; +} + +/*------------------------------------------------- + * zstd_codec_free + *------------------------------------------------- + */ + +void zstd_codec_free(void* codec) +{ + zstd_codec_data* zstd_codec = (zstd_codec_data*) codec; + + ZSTD_freeDStream(zstd_codec->dstream); +} + +/*------------------------------------------------- + * decompress - decompress data using the ZSTD + * codec + *------------------------------------------------- + */ +chd_error zstd_codec_decompress(void* codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + ZSTD_inBuffer input; + ZSTD_outBuffer output; + + /* initialize */ + zstd_codec_data* zstd_codec = (zstd_codec_data*) codec; + + /* reset decompressor */ + size_t zstd_res = ZSTD_initDStream(zstd_codec->dstream); + + input.src = src; + input.size = complen; + input.pos = 0; + + output.dst = dest; + output.size = destlen; + output.pos = 0; + + if (ZSTD_isError(zstd_res)) + { + printf("INITI DSTREAM FAILED!\n"); + return CHDERR_DECOMPRESSION_ERROR; + } + + while ((input.pos < input.size) && (output.pos < output.size)) + { + zstd_res = ZSTD_decompressStream(zstd_codec->dstream, &output, &input); + if (ZSTD_isError(zstd_res)) + { + printf("DECOMPRESSION ERROR IN LOOP\n"); + return CHDERR_DECOMPRESSION_ERROR; + } + } + if (output.pos != output.size) + { + printf("OUTPUT DOESN'T MATCH!\n"); + return CHDERR_DECOMPRESSION_ERROR; + } + return CHDERR_NONE; + +} + +/* cdzs */ +chd_error cdzs_codec_init(void* codec, uint32_t hunkbytes) +{ + chd_error ret; + cdzs_codec_data* cdzs = (cdzs_codec_data*) codec; + + /* allocate buffer */ + cdzs->buffer = (uint8_t*)malloc(sizeof(uint8_t) * hunkbytes); + if (cdzs->buffer == NULL) + return CHDERR_OUT_OF_MEMORY; + + /* make sure the CHD's hunk size is an even multiple of the frame size */ + ret = zstd_codec_init(&cdzs->base_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SECTOR_DATA); + if (ret != CHDERR_NONE) + return ret; + +#ifdef WANT_SUBCODE + ret = zstd_codec_init(&cdzs->subcode_decompressor, (hunkbytes / CD_FRAME_SIZE) * CD_MAX_SUBCODE_DATA); + if (ret != CHDERR_NONE) + return ret; +#endif + + if (hunkbytes % CD_FRAME_SIZE != 0) + return CHDERR_CODEC_ERROR; + + return CHDERR_NONE; +} + +void cdzs_codec_free(void* codec) +{ + cdzs_codec_data* cdzs = (cdzs_codec_data*) codec; + free(cdzs->buffer); + zstd_codec_free(&cdzs->base_decompressor); +#ifdef WANT_SUBCODE + zstd_codec_free(&cdzs->subcode_decompressor); +#endif +} + +chd_error cdzs_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) +{ + uint32_t framenum; + cdzs_codec_data* cdzs = (cdzs_codec_data*)codec; + + /* determine header bytes */ + uint32_t frames = destlen / CD_FRAME_SIZE; + uint32_t complen_bytes = (destlen < 65536) ? 2 : 3; + uint32_t ecc_bytes = (frames + 7) / 8; + uint32_t header_bytes = ecc_bytes + complen_bytes; + + /* extract compressed length of base */ + uint32_t complen_base = (src[ecc_bytes + 0] << 8) | src[ecc_bytes + 1]; + if (complen_bytes > 2) + complen_base = (complen_base << 8) | src[ecc_bytes + 2]; + + /* reset and decode */ + zstd_codec_decompress(&cdzs->base_decompressor, &src[header_bytes], complen_base, &cdzs->buffer[0], frames * CD_MAX_SECTOR_DATA); +#ifdef WANT_SUBCODE + zstd_codec_decompress(&cdzs->subcode_decompressor, &src[header_bytes + complen_base], complen - complen_base - header_bytes, &cdzs->buffer[frames * CD_MAX_SECTOR_DATA], frames * CD_MAX_SUBCODE_DATA); +#endif + + /* reassemble the data */ + for (framenum = 0; framenum < frames; framenum++) + { + uint8_t *sector; + + memcpy(&dest[framenum * CD_FRAME_SIZE], &cdzs->buffer[framenum * CD_MAX_SECTOR_DATA], CD_MAX_SECTOR_DATA); +#ifdef WANT_SUBCODE + memcpy(&dest[framenum * CD_FRAME_SIZE + CD_MAX_SECTOR_DATA], &cdzs->buffer[frames * CD_MAX_SECTOR_DATA + framenum * CD_MAX_SUBCODE_DATA], CD_MAX_SUBCODE_DATA); +#endif + +#ifdef WANT_RAW_DATA_SECTOR + /* reconstitute the ECC data and sync header */ + sector = (uint8_t *)&dest[framenum * CD_FRAME_SIZE]; + if ((src[framenum / 8] & (1 << (framenum % 8))) != 0) + { + memcpy(sector, s_cd_sync_header, sizeof(s_cd_sync_header)); + ecc_generate(sector); + } +#endif + } + return CHDERR_NONE; +} diff --git a/libretro-common/include/libchdr/cdrom.h b/libretro-common/include/libchdr/cdrom.h index 041ff2fa60..4ceb0b0331 100644 --- a/libretro-common/include/libchdr/cdrom.h +++ b/libretro-common/include/libchdr/cdrom.h @@ -14,14 +14,14 @@ #define __CDROM_H__ #include +#include /*************************************************************************** CONSTANTS ***************************************************************************/ /* tracks are padded to a multiple of this many frames */ -#define CD_TRACK_PADDING (4) - +#define CD_TRACK_PADDING (4) #define CD_MAX_TRACKS (99) /* AFAIK the theoretical limit */ #define CD_MAX_SECTOR_DATA (2352) #define CD_MAX_SUBCODE_DATA (96) @@ -66,4 +66,45 @@ void ecc_generate(uint8_t *sector); void ecc_clear(uint8_t *sector); #endif + + +/*************************************************************************** + INLINE FUNCTIONS +***************************************************************************/ + +static INLINE uint32_t msf_to_lba(uint32_t msf) +{ + return ( ((msf&0x00ff0000)>>16) * 60 * 75) + (((msf&0x0000ff00)>>8) * 75) + ((msf&0x000000ff)>>0); +} + +static INLINE uint32_t lba_to_msf(uint32_t lba) +{ + uint8_t m, s, f; + + m = lba / (60 * 75); + lba -= m * (60 * 75); + s = lba / 75; + f = lba % 75; + + return ((m / 10) << 20) | ((m % 10) << 16) | + ((s / 10) << 12) | ((s % 10) << 8) | + ((f / 10) << 4) | ((f % 10) << 0); +} + +/** + * segacd needs it like this.. investigate + * Angelo also says PCE tracks often start playing at the + * wrong address.. related? + **/ +static INLINE uint32_t lba_to_msf_alt(int lba) +{ + uint32_t ret = 0; + + ret |= ((lba / (60 * 75))&0xff)<<16; + ret |= (((lba / 75) % 60)&0xff)<<8; + ret |= ((lba % 75)&0xff)<<0; + + return ret; +} + #endif /* __CDROM_H__ */ diff --git a/libretro-common/include/libchdr/chd.h b/libretro-common/include/libchdr/chd.h index b22b6e4835..72b591a2b6 100644 --- a/libretro-common/include/libchdr/chd.h +++ b/libretro-common/include/libchdr/chd.h @@ -46,8 +46,8 @@ extern "C" { #endif -#include "coretypes.h" -#include +#include +#include /*************************************************************************** @@ -58,67 +58,67 @@ extern "C" { V1 header: [ 0] char tag[8]; // 'MComprHD' - [ 8] UINT32 length; // length of header (including tag and length fields) - [ 12] UINT32 version; // drive format version - [ 16] UINT32 flags; // flags (see below) - [ 20] UINT32 compression; // compression type - [ 24] UINT32 hunksize; // 512-byte sectors per hunk - [ 28] UINT32 totalhunks; // total # of hunks represented - [ 32] UINT32 cylinders; // number of cylinders on hard disk - [ 36] UINT32 heads; // number of heads on hard disk - [ 40] UINT32 sectors; // number of sectors on hard disk - [ 44] UINT8 md5[16]; // MD5 checksum of raw data - [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file + [ 8] uint32_t length; // length of header (including tag and length fields) + [ 12] uint32_t version; // drive format version + [ 16] uint32_t flags; // flags (see below) + [ 20] uint32_t compression; // compression type + [ 24] uint32_t hunksize; // 512-byte sectors per hunk + [ 28] uint32_t totalhunks; // total # of hunks represented + [ 32] uint32_t cylinders; // number of cylinders on hard disk + [ 36] uint32_t heads; // number of heads on hard disk + [ 40] uint32_t sectors; // number of sectors on hard disk + [ 44] uint8_t md5[16]; // MD5 checksum of raw data + [ 60] uint8_t parentmd5[16]; // MD5 checksum of parent file [ 76] (V1 header length) V2 header: [ 0] char tag[8]; // 'MComprHD' - [ 8] UINT32 length; // length of header (including tag and length fields) - [ 12] UINT32 version; // drive format version - [ 16] UINT32 flags; // flags (see below) - [ 20] UINT32 compression; // compression type - [ 24] UINT32 hunksize; // seclen-byte sectors per hunk - [ 28] UINT32 totalhunks; // total # of hunks represented - [ 32] UINT32 cylinders; // number of cylinders on hard disk - [ 36] UINT32 heads; // number of heads on hard disk - [ 40] UINT32 sectors; // number of sectors on hard disk - [ 44] UINT8 md5[16]; // MD5 checksum of raw data - [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file - [ 76] UINT32 seclen; // number of bytes per sector + [ 8] uint32_t length; // length of header (including tag and length fields) + [ 12] uint32_t version; // drive format version + [ 16] uint32_t flags; // flags (see below) + [ 20] uint32_t compression; // compression type + [ 24] uint32_t hunksize; // seclen-byte sectors per hunk + [ 28] uint32_t totalhunks; // total # of hunks represented + [ 32] uint32_t cylinders; // number of cylinders on hard disk + [ 36] uint32_t heads; // number of heads on hard disk + [ 40] uint32_t sectors; // number of sectors on hard disk + [ 44] uint8_t md5[16]; // MD5 checksum of raw data + [ 60] uint8_t parentmd5[16]; // MD5 checksum of parent file + [ 76] uint32_t seclen; // number of bytes per sector [ 80] (V2 header length) V3 header: [ 0] char tag[8]; // 'MComprHD' - [ 8] UINT32 length; // length of header (including tag and length fields) - [ 12] UINT32 version; // drive format version - [ 16] UINT32 flags; // flags (see below) - [ 20] UINT32 compression; // compression type - [ 24] UINT32 totalhunks; // total # of hunks represented - [ 28] UINT64 logicalbytes; // logical size of the data (in bytes) - [ 36] UINT64 metaoffset; // offset to the first blob of metadata - [ 44] UINT8 md5[16]; // MD5 checksum of raw data - [ 60] UINT8 parentmd5[16]; // MD5 checksum of parent file - [ 76] UINT32 hunkbytes; // number of bytes per hunk - [ 80] UINT8 sha1[20]; // SHA1 checksum of raw data - [100] UINT8 parentsha1[20];// SHA1 checksum of parent file + [ 8] uint32_t length; // length of header (including tag and length fields) + [ 12] uint32_t version; // drive format version + [ 16] uint32_t flags; // flags (see below) + [ 20] uint32_t compression; // compression type + [ 24] uint32_t totalhunks; // total # of hunks represented + [ 28] uint64_t logicalbytes; // logical size of the data (in bytes) + [ 36] uint64_t metaoffset; // offset to the first blob of metadata + [ 44] uint8_t md5[16]; // MD5 checksum of raw data + [ 60] uint8_t parentmd5[16]; // MD5 checksum of parent file + [ 76] uint32_t hunkbytes; // number of bytes per hunk + [ 80] uint8_t sha1[20]; // SHA1 checksum of raw data + [100] uint8_t parentsha1[20];// SHA1 checksum of parent file [120] (V3 header length) V4 header: [ 0] char tag[8]; // 'MComprHD' - [ 8] UINT32 length; // length of header (including tag and length fields) - [ 12] UINT32 version; // drive format version - [ 16] UINT32 flags; // flags (see below) - [ 20] UINT32 compression; // compression type - [ 24] UINT32 totalhunks; // total # of hunks represented - [ 28] UINT64 logicalbytes; // logical size of the data (in bytes) - [ 36] UINT64 metaoffset; // offset to the first blob of metadata - [ 44] UINT32 hunkbytes; // number of bytes per hunk - [ 48] UINT8 sha1[20]; // combined raw+meta SHA1 - [ 68] UINT8 parentsha1[20];// combined raw+meta SHA1 of parent - [ 88] UINT8 rawsha1[20]; // raw data SHA1 + [ 8] uint32_t length; // length of header (including tag and length fields) + [ 12] uint32_t version; // drive format version + [ 16] uint32_t flags; // flags (see below) + [ 20] uint32_t compression; // compression type + [ 24] uint32_t totalhunks; // total # of hunks represented + [ 28] uint64_t logicalbytes; // logical size of the data (in bytes) + [ 36] uint64_t metaoffset; // offset to the first blob of metadata + [ 44] uint32_t hunkbytes; // number of bytes per hunk + [ 48] uint8_t sha1[20]; // combined raw+meta SHA1 + [ 68] uint8_t parentsha1[20];// combined raw+meta SHA1 of parent + [ 88] uint8_t rawsha1[20]; // raw data SHA1 [108] (V4 header length) Flags: @@ -130,17 +130,17 @@ extern "C" { V5 header: [ 0] char tag[8]; // 'MComprHD' - [ 8] uint32_t length; // length of header (including tag and length fields) - [ 12] uint32_t version; // drive format version - [ 16] uint32_t compressors[4];// which custom compressors are used? - [ 32] uint64_t logicalbytes; // logical size of the data (in bytes) - [ 40] uint64_t mapoffset; // offset to the map - [ 48] uint64_t metaoffset; // offset to the first blob of metadata - [ 56] uint32_t hunkbytes; // number of bytes per hunk (512k maximum) - [ 60] uint32_t unitbytes; // number of bytes per unit within each hunk - [ 64] uint8_t rawsha1[20]; // raw data SHA1 - [ 84] uint8_t sha1[20]; // combined raw+meta SHA1 - [104] uint8_t parentsha1[20];// combined raw+meta SHA1 of parent + [ 8] uint32_t_t length; // length of header (including tag and length fields) + [ 12] uint32_t_t version; // drive format version + [ 16] uint32_t_t compressors[4];// which custom compressors are used? + [ 32] uint64_t_t logicalbytes; // logical size of the data (in bytes) + [ 40] uint64_t_t mapoffset; // offset to the map + [ 48] uint64_t_t metaoffset; // offset to the first blob of metadata + [ 56] uint32_t_t hunkbytes; // number of bytes per hunk (512k maximum) + [ 60] uint32_t_t unitbytes; // number of bytes per unit within each hunk + [ 64] uint8_t_t rawsha1[20]; // raw data SHA1 + [ 84] uint8_t_t sha1[20]; // combined raw+meta SHA1 + [104] uint8_t_t parentsha1[20];// combined raw+meta SHA1 of parent [124] (V5 header length) If parentsha1 != 0, we have a parent (no need for flags) @@ -148,28 +148,29 @@ extern "C" { V5 uncompressed map format: - [ 0] uint32_t offset; // starting offset / hunk size + [ 0] uint32_t_t offset; // starting offset / hunk size V5 compressed map format header: - [ 0] uint32_t length; // length of compressed map + [ 0] uint32_t_t length; // length of compressed map [ 4] UINT48 datastart; // offset of first block [ 10] uint16_t crc; // crc-16 of the map - [ 12] uint8_t lengthbits; // bits used to encode complength - [ 13] uint8_t hunkbits; // bits used to encode self-refs - [ 14] uint8_t parentunitbits; // bits used to encode parent unit refs - [ 15] uint8_t reserved; // future use + [ 12] uint8_t_t lengthbits; // bits used to encode complength + [ 13] uint8_t_t hunkbits; // bits used to encode self-refs + [ 14] uint8_t_t parentunitbits; // bits used to encode parent unit refs + [ 15] uint8_t_t reserved; // future use [ 16] (compressed header length) Each compressed map entry, once expanded, looks like: - [ 0] uint8_t compression; // compression type + [ 0] uint8_t_t compression; // compression type [ 1] UINT24 complength; // compressed length [ 4] UINT48 offset; // offset [ 10] uint16_t crc; // crc-16 of the data ***************************************************************************/ + /*************************************************************************** CONSTANTS ***************************************************************************/ @@ -201,12 +202,17 @@ extern "C" { #define CHDCOMPRESSION_ZLIB_PLUS 2 #define CHDCOMPRESSION_AV 3 +#define CHD_CODEC_NONE 0 #define CHD_CODEC_ZLIB CHD_MAKE_TAG('z','l','i','b') - +#define CHD_CODEC_LZMA CHD_MAKE_TAG('l','z','m','a') +#define CHD_CODEC_HUFFMAN CHD_MAKE_TAG('h','u','f','f') +#define CHD_CODEC_FLAC CHD_MAKE_TAG('f','l','a','c') +#define CHD_CODEC_ZSTD CHD_MAKE_TAG('z', 's', 't', 'd') /* general codecs with CD frontend */ -#define CHD_CODEC_CD_ZLIB CHD_MAKE_TAG('c','d','z','l') -#define CHD_CODEC_CD_LZMA CHD_MAKE_TAG('c','d','l','z') -#define CHD_CODEC_CD_FLAC CHD_MAKE_TAG('c','d','f','l') +#define CHD_CODEC_CD_ZLIB CHD_MAKE_TAG('c','d','z','l') +#define CHD_CODEC_CD_LZMA CHD_MAKE_TAG('c','d','l','z') +#define CHD_CODEC_CD_FLAC CHD_MAKE_TAG('c','d','f','l') +#define CHD_CODEC_CD_ZSTD CHD_MAKE_TAG('c','d','z','s') /* A/V codec configuration parameters */ #define AV_CODEC_COMPRESS_CONFIG 1 @@ -214,14 +220,14 @@ extern "C" { /* metadata parameters */ #define CHDMETATAG_WILDCARD 0 -#define CHD_METAINDEX_APPEND ((UINT32)-1) +#define CHD_METAINDEX_APPEND ((uint32_t)-1) /* metadata flags */ #define CHD_MDFLAGS_CHECKSUM 0x01 /* indicates data is checksummed */ /* standard hard disk metadata */ #define HARD_DISK_METADATA_TAG CHD_MAKE_TAG('G','D','D','D') -#define HARD_DISK_METADATA_FORMAT "CYLS:%u,HEADS:%u,SECS:%u,BPS:%u" +#define HARD_DISK_METADATA_FORMAT "CYLS:%d,HEADS:%d,SECS:%d,BPS:%d" /* hard disk identify information */ #define HARD_DISK_IDENT_METADATA_TAG CHD_MAKE_TAG('I','D','N','T') @@ -234,13 +240,13 @@ extern "C" { /* standard CD-ROM metadata */ #define CDROM_OLD_METADATA_TAG CHD_MAKE_TAG('C','H','C','D') -#define CDROM_TRACK_METADATA_TAG CHD_MAKE_TAG('C','H','T','R') -#define CDROM_TRACK_METADATA_FORMAT "TRACK:%u TYPE:%s SUBTYPE:%s FRAMES:%u" +#define CDROM_TRACK_METADATA_TAG CHD_MAKE_TAG('C','H','T','R') +#define CDROM_TRACK_METADATA_FORMAT "TRACK:%d TYPE:%s SUBTYPE:%s FRAMES:%d" #define CDROM_TRACK_METADATA2_TAG CHD_MAKE_TAG('C','H','T','2') -#define CDROM_TRACK_METADATA2_FORMAT "TRACK:%u TYPE:%s SUBTYPE:%s FRAMES:%u PREGAP:%u PGTYPE:%s PGSUB:%s POSTGAP:%u" +#define CDROM_TRACK_METADATA2_FORMAT "TRACK:%d TYPE:%s SUBTYPE:%s FRAMES:%d PREGAP:%d PGTYPE:%s PGSUB:%s POSTGAP:%d" #define GDROM_OLD_METADATA_TAG CHD_MAKE_TAG('C','H','G','T') #define GDROM_TRACK_METADATA_TAG CHD_MAKE_TAG('C', 'H', 'G', 'D') -#define GDROM_TRACK_METADATA_FORMAT "TRACK:%u TYPE:%s SUBTYPE:%s FRAMES:%u PAD:%u PREGAP:%u PGTYPE:%s PGSUB:%s POSTGAP:%u" +#define GDROM_TRACK_METADATA_FORMAT "TRACK:%d TYPE:%s SUBTYPE:%s FRAMES:%d PAD:%d PREGAP:%d PGTYPE:%s PGSUB:%s POSTGAP:%d" /* standard A/V metadata */ #define AV_METADATA_TAG CHD_MAKE_TAG('A','V','A','V') @@ -249,6 +255,9 @@ extern "C" { /* A/V laserdisc frame metadata */ #define AV_LD_METADATA_TAG CHD_MAKE_TAG('A','V','L','D') +/* DVD metadata */ +#define DVD_METADATA_TAG CHD_MAKE_TAG('D','V','D',' ') + /* CHD open values */ #define CHD_OPEN_READ 1 #define CHD_OPEN_READWRITE 2 @@ -287,6 +296,8 @@ enum _chd_error }; typedef enum _chd_error chd_error; + + /*************************************************************************** TYPE DEFINITIONS ***************************************************************************/ @@ -294,101 +305,129 @@ typedef enum _chd_error chd_error; /* opaque types */ typedef struct _chd_file chd_file; + /* extract header structure (NOT the on-disk header structure) */ typedef struct _chd_header chd_header; struct _chd_header { - UINT32 length; /* length of header data */ - UINT32 version; /* drive format version */ - UINT32 flags; /* flags field */ - UINT32 compression[4]; /* compression type */ - UINT32 hunkbytes; /* number of bytes per hunk */ - UINT32 totalhunks; /* total # of hunks represented */ - UINT64 logicalbytes; /* logical size of the data */ - UINT64 metaoffset; /* offset in file of first metadata */ - UINT64 mapoffset; /* TOOD V5 */ - UINT8 md5[CHD_MD5_BYTES]; /* overall MD5 checksum */ - UINT8 parentmd5[CHD_MD5_BYTES]; /* overall MD5 checksum of parent */ - UINT8 sha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum */ - UINT8 rawsha1[CHD_SHA1_BYTES]; /* SHA1 checksum of raw data */ - UINT8 parentsha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum of parent */ - UINT32 unitbytes; /* TODO V5 */ - UINT64 unitcount; /* TODO V5 */ - UINT32 hunkcount; /* TODO V5 */ + uint32_t length; /* length of header data */ + uint32_t version; /* drive format version */ + uint32_t flags; /* flags field */ + uint32_t compression[4]; /* compression type */ + uint32_t hunkbytes; /* number of bytes per hunk */ + uint32_t totalhunks; /* total # of hunks represented */ + uint64_t logicalbytes; /* logical size of the data */ + uint64_t metaoffset; /* offset in file of first metadata */ + uint64_t mapoffset; /* TOOD V5 */ + uint8_t md5[CHD_MD5_BYTES]; /* overall MD5 checksum */ + uint8_t parentmd5[CHD_MD5_BYTES]; /* overall MD5 checksum of parent */ + uint8_t sha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum */ + uint8_t rawsha1[CHD_SHA1_BYTES]; /* SHA1 checksum of raw data */ + uint8_t parentsha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum of parent */ + uint32_t unitbytes; /* TODO V5 */ + uint64_t unitcount; /* TODO V5 */ + uint32_t hunkcount; /* TODO V5 */ /* map information */ - UINT32 mapentrybytes; /* length of each entry in a map (V5) */ - UINT8* rawmap; /* raw map data */ + uint32_t mapentrybytes; /* length of each entry in a map (V5) */ + uint8_t* rawmap; /* raw map data */ - UINT32 obsolete_cylinders; /* obsolete field -- do not use! */ - UINT32 obsolete_sectors; /* obsolete field -- do not use! */ - UINT32 obsolete_heads; /* obsolete field -- do not use! */ - UINT32 obsolete_hunksize; /* obsolete field -- do not use! */ + uint32_t obsolete_cylinders; /* obsolete field -- do not use! */ + uint32_t obsolete_sectors; /* obsolete field -- do not use! */ + uint32_t obsolete_heads; /* obsolete field -- do not use! */ + uint32_t obsolete_hunksize; /* obsolete field -- do not use! */ }; + /* structure for returning information about a verification pass */ typedef struct _chd_verify_result chd_verify_result; struct _chd_verify_result { - UINT8 md5[CHD_MD5_BYTES]; /* overall MD5 checksum */ - UINT8 sha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum */ - UINT8 rawsha1[CHD_SHA1_BYTES]; /* SHA1 checksum of raw data */ - UINT8 metasha1[CHD_SHA1_BYTES]; /* SHA1 checksum of metadata */ + uint8_t md5[CHD_MD5_BYTES]; /* overall MD5 checksum */ + uint8_t sha1[CHD_SHA1_BYTES]; /* overall SHA1 checksum */ + uint8_t rawsha1[CHD_SHA1_BYTES]; /* SHA1 checksum of raw data */ + uint8_t metasha1[CHD_SHA1_BYTES]; /* SHA1 checksum of metadata */ }; + + /*************************************************************************** FUNCTION PROTOTYPES ***************************************************************************/ +#ifdef _MSC_VER +#ifdef CHD_DLL +#ifdef CHD_DLL_EXPORTS +#define CHD_EXPORT __declspec(dllexport) +#else +#define CHD_EXPORT __declspec(dllimport) +#endif +#else +#define CHD_EXPORT +#endif +#else +#define CHD_EXPORT __attribute__ ((visibility("default"))) +#endif + /* ----- CHD file management ----- */ /* create a new CHD file fitting the given description */ -/* chd_error chd_create(const char *filename, UINT64 logicalbytes, UINT32 hunkbytes, UINT32 compression, chd_file *parent); */ +/* chd_error chd_create(const char *filename, uint64_t logicalbytes, uint32_t hunkbytes, uint32_t compression, chd_file *parent); */ /* same as chd_create(), but accepts an already-opened core_file object */ -/* chd_error chd_create_file(core_file *file, UINT64 logicalbytes, UINT32 hunkbytes, UINT32 compression, chd_file *parent); */ +/* chd_error chd_create_file(core_file *file, uint64_t logicalbytes, uint32_t hunkbytes, uint32_t compression, chd_file *parent); */ /* open an existing CHD file */ -chd_error chd_open_file(RFILE *file, int mode, chd_file *parent, chd_file **chd); - -chd_error chd_open(const char *filename, int mode, chd_file *parent, chd_file **chd); +CHD_EXPORT chd_error chd_open_core_file(core_file *file, int mode, chd_file *parent, chd_file **chd); +CHD_EXPORT chd_error chd_open_file(FILE *file, int mode, chd_file *parent, chd_file **chd); +CHD_EXPORT chd_error chd_open(const char *filename, int mode, chd_file *parent, chd_file **chd); /* precache underlying file */ -chd_error chd_precache(chd_file *chd); +CHD_EXPORT chd_error chd_precache(chd_file *chd); /* close a CHD file */ -void chd_close(chd_file *chd); +CHD_EXPORT void chd_close(chd_file *chd); /* return the associated core_file */ -RFILE *chd_core_file(chd_file *chd); +CHD_EXPORT core_file *chd_core_file(chd_file *chd); /* return an error string for the given CHD error */ -const char *chd_error_string(chd_error err); +CHD_EXPORT const char *chd_error_string(chd_error err); + + /* ----- CHD header management ----- */ /* return a pointer to the extracted CHD header data */ -const chd_header *chd_get_header(chd_file *chd); +CHD_EXPORT const chd_header *chd_get_header(chd_file *chd); + +/* read CHD header data from file into the pointed struct */ +CHD_EXPORT chd_error chd_read_header(const char *filename, chd_header *header); + + /* ----- core data read/write ----- */ /* read one hunk from the CHD file */ -chd_error chd_read(chd_file *chd, UINT32 hunknum, void *buffer); +CHD_EXPORT chd_error chd_read(chd_file *chd, uint32_t hunknum, void *buffer); + + /* ----- metadata management ----- */ /* get indexed metadata of a particular sort */ -chd_error chd_get_metadata(chd_file *chd, UINT32 searchtag, UINT32 searchindex, void *output, UINT32 outputlen, UINT32 *resultlen, UINT32 *resulttag, UINT8 *resultflags); +CHD_EXPORT chd_error chd_get_metadata(chd_file *chd, uint32_t searchtag, uint32_t searchindex, void *output, uint32_t outputlen, uint32_t *resultlen, uint32_t *resulttag, uint8_t *resultflags); + + + /* ----- codec interfaces ----- */ /* set internal codec parameters */ -chd_error chd_codec_config(chd_file *chd, int param, void *config); +CHD_EXPORT chd_error chd_codec_config(chd_file *chd, int param, void *config); /* return a string description of a codec */ -const char *chd_get_codec_name(UINT32 codec); - -extern const uint8_t s_cd_sync_header[12]; +CHD_EXPORT const char *chd_get_codec_name(uint32_t codec); #ifdef __cplusplus } diff --git a/libretro-common/include/libchdr/chdconfig.h b/libretro-common/include/libchdr/chdconfig.h new file mode 100644 index 0000000000..3e2ee03c1c --- /dev/null +++ b/libretro-common/include/libchdr/chdconfig.h @@ -0,0 +1,12 @@ +#ifndef __CHDCONFIG_H__ +#define __CHDCONFIG_H__ + +#include + +/* Configure CHDR features here */ +#define WANT_RAW_DATA_SECTOR 1 +#define WANT_SUBCODE 1 +/*#define NEED_CACHE_HUNK 1*/ +/*#define VERIFY_BLOCK_CRC 1*/ + +#endif diff --git a/libretro-common/include/libchdr/coretypes.h b/libretro-common/include/libchdr/coretypes.h index 2a68a8a4c3..8f687c1548 100644 --- a/libretro-common/include/libchdr/coretypes.h +++ b/libretro-common/include/libchdr/coretypes.h @@ -4,6 +4,13 @@ #include #include #include +#ifdef USE_LIBRETRO_VFS +#include +#endif + +#ifdef WANT_RAW_DATA_SECTOR +extern const uint8_t s_cd_sync_header[12]; +#endif typedef uint64_t UINT64; #ifndef OSD_CPU_H @@ -19,4 +26,54 @@ typedef int16_t INT16; typedef int8_t INT8; #endif +#ifndef ARRAY_LENGTH +#define ARRAY_LENGTH(x) (sizeof(x)/sizeof(x[0])) +#endif + +typedef struct chd_core_file { + /* + * arbitrary pointer to data the implementation uses to implement the below functions + */ + void *argp; + + /* + * return the size of a given file as a 64-bit unsigned integer. + * the position of the file pointer after calling this function is + * undefined because many implementations will seek to the end of the + * file and call ftell. + * + * on error, (uint64_t)-1 is returned. + */ + uint64_t(*fsize)(struct chd_core_file*); + + /* + * should match the behavior of fread, except the FILE* argument at the end + * will be replaced with a struct chd_core_file*. + */ + size_t(*fread)(void*,size_t,size_t,struct chd_core_file*); + + /* closes the given file. */ + int (*fclose)(struct chd_core_file*); + + /* fseek clone. */ + int (*fseek)(struct chd_core_file*, int64_t, int); +} core_file; + +static INLINE int core_fclose(core_file *fp) { + return fp->fclose(fp); +} + +static INLINE size_t core_fread(core_file *fp, void *ptr, size_t len) { + return fp->fread(ptr, 1, len, fp); +} + +static INLINE int core_fseek(core_file* fp, int64_t offset, int whence) { + return fp->fseek(fp, offset, whence); +} + +static INLINE uint64_t core_fsize(core_file *fp) +{ + return fp->fsize(fp); +} + #endif diff --git a/libretro-common/include/libchdr/flac.h b/libretro-common/include/libchdr/flac.h index 2ce112d27a..7168ddf2e8 100644 --- a/libretro-common/include/libchdr/flac.h +++ b/libretro-common/include/libchdr/flac.h @@ -14,9 +14,8 @@ #define __FLAC_H__ #include -#include "libchdr_zlib.h" -#include "FLAC/ordinals.h" -#include "FLAC/stream_decoder.h" + +#include /*************************************************************************** * TYPE DEFINITIONS @@ -26,14 +25,14 @@ typedef struct _flac_decoder flac_decoder; struct _flac_decoder { /* output state */ - FLAC__StreamDecoder* decoder; /* actual encoder */ + void * decoder; /* actual encoder */ uint32_t sample_rate; /* decoded sample rate */ uint8_t channels; /* decoded number of channels */ uint8_t bits_per_sample; /* decoded bits per sample */ uint32_t compressed_offset; /* current offset in compressed data */ - const FLAC__byte * compressed_start; /* start of compressed data */ + const uint8_t * compressed_start; /* start of compressed data */ uint32_t compressed_length; /* length of compressed data */ - const FLAC__byte * compressed2_start; /* start of compressed data */ + const uint8_t * compressed2_start; /* start of compressed data */ uint32_t compressed2_length; /* length of compressed data */ int16_t * uncompressed_start[8]; /* pointer to start of uncompressed data (up to 8 streams) */ uint32_t uncompressed_offset; /* current position in uncompressed data */ @@ -44,12 +43,20 @@ struct _flac_decoder { /* ======================> flac_decoder */ -void flac_decoder_init(flac_decoder* decoder); +int flac_decoder_init(flac_decoder* decoder); void flac_decoder_free(flac_decoder* decoder); int flac_decoder_reset(flac_decoder* decoder, uint32_t sample_rate, uint8_t num_channels, uint32_t block_size, const void *buffer, uint32_t length); int flac_decoder_decode_interleaved(flac_decoder* decoder, int16_t *samples, uint32_t num_samples, int swap_endian); uint32_t flac_decoder_finish(flac_decoder* decoder); +/* codec-private data for the FLAC codec */ +typedef struct _flac_codec_data flac_codec_data; +struct _flac_codec_data { + /* internal state */ + int native_endian; + flac_decoder decoder; +}; + /* codec-private data for the CDFL codec */ typedef struct _cdfl_codec_data cdfl_codec_data; struct _cdfl_codec_data { @@ -62,9 +69,4 @@ struct _cdfl_codec_data { uint8_t* buffer; }; -/* cdfl compression codec */ -chd_error cdfl_codec_init(void* codec, uint32_t hunkbytes); -void cdfl_codec_free(void* codec); -chd_error cdfl_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); - #endif /* __FLAC_H__ */ diff --git a/libretro-common/include/libchdr/huffman.h b/libretro-common/include/libchdr/huffman.h index ec5ef54a7b..6c9f511368 100644 --- a/libretro-common/include/libchdr/huffman.h +++ b/libretro-common/include/libchdr/huffman.h @@ -13,7 +13,8 @@ #ifndef __HUFFMAN_H__ #define __HUFFMAN_H__ -#include "bitstream.h" +#include + /*************************************************************************** * CONSTANTS diff --git a/libretro-common/include/libchdr/libchdr_zlib.h b/libretro-common/include/libchdr/libchdr_zlib.h index 86544d6d00..e01b024db3 100644 --- a/libretro-common/include/libchdr/libchdr_zlib.h +++ b/libretro-common/include/libchdr/libchdr_zlib.h @@ -26,8 +26,8 @@ typedef struct _zlib_allocator zlib_allocator; struct _zlib_allocator { - UINT32 * allocptr[MAX_ZLIB_ALLOCS]; - UINT32 * allocptr2[MAX_ZLIB_ALLOCS]; + uint32_t * allocptr[MAX_ZLIB_ALLOCS]; + uint32_t * allocptr2[MAX_ZLIB_ALLOCS]; }; typedef struct _zlib_codec_data zlib_codec_data; @@ -48,6 +48,11 @@ struct _cdzl_codec_data { uint8_t* buffer; }; +extern chd_error zlib_codec_init(void *codec, uint32_t hunkbytes); +extern void zlib_codec_free(void *codec); +extern chd_error zlib_codec_decompress(void *codec, const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen); +extern void zlib_allocator_free(voidpf opaque); + /* zlib compression codec */ chd_error zlib_codec_init(void *codec, uint32_t hunkbytes); diff --git a/libretro-common/include/libchdr/libchdr_zstd.h b/libretro-common/include/libchdr/libchdr_zstd.h new file mode 100644 index 0000000000..5849d68d22 --- /dev/null +++ b/libretro-common/include/libchdr/libchdr_zstd.h @@ -0,0 +1,38 @@ +/* license:BSD-3-Clause + * copyright-holders:Aaron Giles + *************************************************************************** + + libchr_zstd.h + + Zstd compression wrappers + +***************************************************************************/ + +#pragma once + +#ifndef __LIBCHDR_ZSTD_H__ +#define __LIBCHDR_ZSTD_H__ + +#include + +#include +#include "coretypes.h" +#include "chd.h" + +typedef struct _zstd_codec_data zstd_codec_data; +struct _zstd_codec_data +{ + ZSTD_DStream *dstream; +}; + +typedef struct _cdzs_codec_data cdzs_codec_data; +struct _cdzs_codec_data +{ + zstd_codec_data base_decompressor; +#ifdef WANT_SUBCODE + zstd_codec_data subcode_decompressor; +#endif + uint8_t* buffer; +}; + +#endif /* __LIBCHDR_ZSTD_H__ */ diff --git a/libretro-common/streams/chd_stream.c b/libretro-common/streams/chd_stream.c index daa718fbca..ec273f5825 100644 --- a/libretro-common/streams/chd_stream.c +++ b/libretro-common/streams/chd_stream.c @@ -31,7 +31,8 @@ #include #include -#define SECTOR_SIZE 2352 +#define SECTOR_RAW_SIZE 2352 +#define SECTOR_SIZE 2048 #define SUBCODE_SIZE 96 #define TRACK_PAD 4 @@ -128,6 +129,16 @@ chdstream_get_meta(chd_file *chd, int idx, metadata_t *md) return true; } + err = chd_get_metadata(chd, DVD_METADATA_TAG, idx, meta, + sizeof(meta), &meta_size, NULL, NULL); + + if (err == CHDERR_NONE) + { + md->track = 1; + strlcpy(md->type, "DVD", sizeof(md->type)); + return true; + } + return false; } @@ -243,14 +254,21 @@ chdstream_t *chdstream_open(const char *path, int32_t track) stream->hunkmem = hunkmem; if (string_is_equal(meta.type, "MODE1_RAW")) - stream->frame_size = SECTOR_SIZE; + stream->frame_size = SECTOR_RAW_SIZE; else if (string_is_equal(meta.type, "MODE2_RAW")) + stream->frame_size = SECTOR_RAW_SIZE; + else if (string_is_equal(meta.type, "MODE1")) stream->frame_size = SECTOR_SIZE; else if (string_is_equal(meta.type, "AUDIO")) { - stream->frame_size = SECTOR_SIZE; + stream->frame_size = SECTOR_RAW_SIZE; stream->swab = true; } + else if (string_is_equal(meta.type, "DVD")) + { + stream->frame_size = hd->unitbytes; + meta.frames = hd->totalhunks; + } else stream->frame_size = hd->unitbytes; diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index 686d6c3678..bbb6f11fbc 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -2410,6 +2410,9 @@ static unsigned menu_displaylist_parse_system_info(file_list_t *list) #ifdef HAVE_7ZIP {SUPPORTS_7ZIP , MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_7ZIP_SUPPORT}, #endif +#ifdef HAVE_ZSTD + {SUPPORTS_ZSTD , MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_ZSTD_SUPPORT}, +#endif #ifdef HAVE_DYLIB {SUPPORTS_DYLIB , MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_DYLIB_SUPPORT}, #endif diff --git a/msg_hash.h b/msg_hash.h index ed2f8a5649..c09e980813 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -3547,6 +3547,7 @@ enum msg_hash_enums MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_XAUDIO2_SUPPORT, MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_ZLIB_SUPPORT, MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_7ZIP_SUPPORT, + MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_ZSTD_SUPPORT, MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_DYLIB_SUPPORT, MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_CG_SUPPORT, MENU_ENUM_LABEL_VALUE_SYSTEM_INFO_GLSL_SUPPORT, diff --git a/qb/config.params.sh b/qb/config.params.sh index 5799e54b73..bc6b4381c7 100644 --- a/qb/config.params.sh +++ b/qb/config.params.sh @@ -142,6 +142,7 @@ HAVE_FLOATSOFTFP=no # Force soft float ABI (for ARM) HAVE_CHD=yes # Compile in chd support CXX_CHD=no HAVE_7ZIP=yes # Compile in 7z support +HAVE_ZSTD=yes # Compile in Zstandard support HAVE_FLAC=auto # Compile in flac support HAVE_DR_MP3=yes # Compile in Dr. MP3 support HAVE_BUILTINFLAC=auto # Bake in flac support diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 5c9828ec48..1daf836365 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -60,6 +60,7 @@ static struct magic_entry MAGIC_NUMBERS[] = { { "Sony - PlayStation", "Sony Computer ", 0x0024f8}, /* PS1 CD license string, PS2 CD doesnt have this string */ { "Sony - PlayStation 2", "PLAYSTATION", 0x009320}, /* PS1 CD and PS2 CD */ { "Sony - PlayStation 2", "PLAYSTATION", 0x008008}, /* PS2 DVD */ + { "Sony - PlayStation 2", " ", 0x008008}, /* PS2 DVD */ { "Sony - PlayStation Portable", "PSP GAME", 0x008008}, { NULL, NULL, 0} }; @@ -284,7 +285,7 @@ int detect_ps1_game(intfstream_t *fd, char *s, size_t len, const char *filename) int detect_ps2_game(intfstream_t *fd, char *s, size_t len, const char *filename) { - #define DISC_DATA_SIZE_PS2 0x84000 + #define DISC_DATA_SIZE_PS2 600000 int pos; char raw_game_id[50]; char *disc_data; @@ -382,6 +383,11 @@ int detect_ps2_game(intfstream_t *fd, char *s, size_t len, const char *filename) raw_game_id[8] = raw_game_id[9]; raw_game_id[9] = raw_game_id[10]; } + /* Wild character conversions */ + if (raw_game_id[8] == 18) + raw_game_id[8] = 51; + if (raw_game_id[9] == 18) + raw_game_id[9] = 51; raw_game_id[10] = '\0'; string_remove_all_whitespace(s, raw_game_id); @@ -410,7 +416,7 @@ int detect_ps2_game(intfstream_t *fd, char *s, size_t len, const char *filename) int detect_psp_game(intfstream_t *fd, char *s, size_t len, const char *filename) { - #define DISC_DATA_SIZE_PSP 40000 + #define DISC_DATA_SIZE_PSP 300000 int pos; char *disc_data = malloc(DISC_DATA_SIZE_PSP);