diff --git a/Makefile.common b/Makefile.common index 1944d4563a..f3cfaf753a 100644 --- a/Makefile.common +++ b/Makefile.common @@ -977,7 +977,9 @@ endif # Compression/Archive -OBJ += $(LIBRETRO_COMM_DIR)/file/archive_file.o +OBJ += $(LIBRETRO_COMM_DIR)/file/archive_file.o \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream.o \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream_pipe.o ifeq ($(HAVE_7ZIP),1) CFLAGS += -I$(DEPS_DIR)/7zip @@ -1003,7 +1005,8 @@ endif ifeq ($(HAVE_ZLIB), 1) - OBJ += $(LIBRETRO_COMM_DIR)/file/archive_file_zlib.o + OBJ += $(LIBRETRO_COMM_DIR)/file/archive_file_zlib.o \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream_zlib.o OBJ += $(ZLIB_OBJS) DEFINES += -DHAVE_ZLIB HAVE_COMPRESSION = 1 diff --git a/griffin/griffin.c b/griffin/griffin.c index 0173a7dea0..b57420f4a5 100644 --- a/griffin/griffin.c +++ b/griffin/griffin.c @@ -64,6 +64,16 @@ ARCHIVE FILE #include "../libretro-common/file/archive_file_7z.c" #endif +/*============================================================ +COMPRESSION +============================================================ */ +#include "../libretro-common/streams/trans_stream.c" +#include "../libretro-common/streams/trans_stream_pipe.c" + +#ifdef HAVE_ZLIB +#include "../libretro-common/streams/trans_stream_zlib.c" +#endif + /*============================================================ ENCODINGS ============================================================ */ diff --git a/libretro-common/file/archive_file_7z.c b/libretro-common/file/archive_file_7z.c index 992b2cbffd..b1234eabc9 100644 --- a/libretro-common/file/archive_file_7z.c +++ b/libretro-common/file/archive_file_7z.c @@ -424,17 +424,8 @@ static uint32_t sevenzip_stream_crc32_calculate(uint32_t crc, const struct file_archive_file_backend sevenzip_backend = { sevenzip_stream_new, sevenzip_stream_free, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, sevenzip_stream_decompress_data_to_file_init, sevenzip_stream_decompress_data_to_file_iterate, - NULL, - NULL, - NULL, sevenzip_stream_crc32_calculate, sevenzip_file_read, sevenzip_parse_file_init, diff --git a/libretro-common/file/archive_file_zlib.c b/libretro-common/file/archive_file_zlib.c index 0269dea358..d2176d0bc7 100644 --- a/libretro-common/file/archive_file_zlib.c +++ b/libretro-common/file/archive_file_zlib.c @@ -22,13 +22,16 @@ #include -#include #include #include +#include #include #include #include +/* Only for MAX_WBITS */ +#include + #ifndef CENTRAL_FILE_HEADER_SIGNATURE #define CENTRAL_FILE_HEADER_SIGNATURE 0x02014b50 #endif @@ -37,107 +40,14 @@ #define END_OF_CENTRAL_DIR_SIGNATURE 0x06054b50 #endif -static void* zlib_stream_new(void) +static void *zlib_stream_new(void) { - return (z_stream*)calloc(1, sizeof(z_stream)); + return zlib_inflate_backend.stream_new(); } -static void zlib_stream_free(void *data) +static void zlib_stream_free(void *stream) { - z_stream *ret = (z_stream*)data; - if (ret) - inflateEnd(ret); -} - -static void zlib_stream_set(void *data, - uint32_t avail_in, - uint32_t avail_out, - const uint8_t *next_in, - uint8_t *next_out - ) -{ - z_stream *stream = (z_stream*)data; - - if (!stream) - return; - - stream->avail_in = avail_in; - stream->avail_out = avail_out; - - stream->next_in = (uint8_t*)next_in; - stream->next_out = next_out; -} - -static uint32_t zlib_stream_get_avail_in(void *data) -{ - z_stream *stream = (z_stream*)data; - - if (!stream) - return 0; - - return stream->avail_in; -} - -static uint32_t zlib_stream_get_avail_out(void *data) -{ - z_stream *stream = (z_stream*)data; - - if (!stream) - return 0; - - return stream->avail_out; -} - -static uint64_t zlib_stream_get_total_out(void *data) -{ - z_stream *stream = (z_stream*)data; - - if (!stream) - return 0; - - return stream->total_out; -} - -static void zlib_stream_decrement_total_out(void *data, unsigned subtraction) -{ - z_stream *stream = (z_stream*)data; - - if (stream) - stream->total_out -= subtraction; -} - -static void zlib_stream_compress_free(void *data) -{ - z_stream *ret = (z_stream*)data; - if (ret) - deflateEnd(ret); -} - -static int zlib_stream_compress_data_to_file(void *data) -{ - int zstatus; - z_stream *stream = (z_stream*)data; - - if (!stream) - return -1; - - zstatus = deflate(stream, Z_FINISH); - - if (zstatus == Z_STREAM_END) - return 1; - - return 0; -} - -static bool zlib_stream_decompress_init(void *data) -{ - z_stream *stream = (z_stream*)data; - - if (!stream) - return false; - if (inflateInit(stream) != Z_OK) - return false; - return true; + zlib_inflate_backend.stream_free(stream); } static bool zlib_stream_decompress_data_to_file_init( @@ -147,59 +57,51 @@ static bool zlib_stream_decompress_data_to_file_init( if (!handle) return false; - if (!(handle->stream = (z_stream*)zlib_stream_new())) + if (!(handle->stream = zlib_inflate_backend.stream_new())) goto error; - if (inflateInit2((z_streamp)handle->stream, -MAX_WBITS) != Z_OK) - goto error; + if (zlib_inflate_backend.define) + zlib_inflate_backend.define(handle->stream, "window_bits", -MAX_WBITS); handle->data = (uint8_t*)malloc(size); if (!handle->data) goto error; - zlib_stream_set(handle->stream, csize, size, - (const uint8_t*)cdata, handle->data); + zlib_inflate_backend.set_in(handle->stream, + (const uint8_t*)cdata, csize); + zlib_inflate_backend.set_out(handle->stream, + handle->data, size); return true; error: - zlib_stream_free(handle->stream); - free(handle->stream); + if (handle->stream) + zlib_inflate_backend.stream_free(handle->stream); if (handle->data) free(handle->data); return false; } -static int zlib_stream_decompress_data_to_file_iterate(void *data) +static int zlib_stream_decompress_data_to_file_iterate(void *stream) { - int zstatus; - z_stream *stream = (z_stream*)data; + bool zstatus; + uint32_t rd, wn; + enum trans_stream_error terror; if (!stream) - goto error; + return -1; - zstatus = inflate(stream, Z_NO_FLUSH); + zstatus = zlib_inflate_backend.trans(stream, false, &rd, &wn, &terror); - if (zstatus == Z_STREAM_END) + if (!zstatus && terror != TRANS_STREAM_ERROR_BUFFER_FULL) + return -1; + + if (zstatus && !terror) return 1; - if (zstatus != Z_OK && zstatus != Z_BUF_ERROR) - goto error; - return 0; - -error: - return -1; -} - -static void zlib_stream_compress_init(void *data, int level) -{ - z_stream *stream = (z_stream*)data; - - if (stream) - deflateInit(stream, level); } static uint32_t zlib_stream_crc32_calculate(uint32_t crc, @@ -462,17 +364,8 @@ static int zip_parse_file_iterate_step(file_archive_transfer_t *state, const struct file_archive_file_backend zlib_backend = { zlib_stream_new, zlib_stream_free, - zlib_stream_set, - zlib_stream_get_avail_in, - zlib_stream_get_avail_out, - zlib_stream_get_total_out, - zlib_stream_decrement_total_out, - zlib_stream_decompress_init, zlib_stream_decompress_data_to_file_init, zlib_stream_decompress_data_to_file_iterate, - zlib_stream_compress_init, - zlib_stream_compress_free, - zlib_stream_compress_data_to_file, zlib_stream_crc32_calculate, zip_file_read, zip_parse_file_init, diff --git a/libretro-common/formats/png/rpng.c b/libretro-common/formats/png/rpng.c index e1204414f0..2ad8c9017d 100644 --- a/libretro-common/formats/png/rpng.c +++ b/libretro-common/formats/png/rpng.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include "rpng_internal.h" @@ -112,7 +112,8 @@ struct rpng_process unsigned pos; } pass; void *stream; - const struct file_archive_file_backend *stream_backend; + size_t avail_in, avail_out, total_out; + const struct trans_stream_backend *stream_backend; }; struct rpng @@ -538,7 +539,7 @@ static int png_reverse_filter_init(const struct png_ihdr *ihdr, png_pass_geom(&pngp->ihdr, pngp->pass.width, pngp->pass.height, NULL, NULL, &pngp->pass.size); - if (pngp->pass.size > pngp->stream_backend->stream_get_total_out(pngp->stream)) + if (pngp->pass.size > pngp->total_out) { free(pngp->data); return -1; @@ -554,7 +555,7 @@ static int png_reverse_filter_init(const struct png_ihdr *ihdr, png_pass_geom(ihdr, ihdr->width, ihdr->height, &pngp->bpp, &pngp->pitch, &pass_size); - if (pngp->stream_backend->stream_get_total_out(pngp->stream) < pass_size) + if (pngp->total_out < pass_size) return -1; pngp->restore_buf_size = 0; @@ -711,7 +712,7 @@ static int png_reverse_filter_adam7_iterate(uint32_t **data_, pngp->inflate_buf += pngp->pass.size; pngp->adam7_restore_buf_size += pngp->pass.size; - pngp->stream_backend->stream_decrement_total_out(pngp->stream, pngp->pass.size); + pngp->total_out -= pngp->pass.size; png_reverse_filter_adam7_deinterlace_pass(data, ihdr, pngp->data, pngp->pass.width, pngp->pass.height, &passes[pngp->pass.pos]); @@ -768,30 +769,31 @@ static int png_reverse_filter_iterate(rpng_t *rpng, uint32_t **data) static int rpng_load_image_argb_process_inflate_init(rpng_t *rpng, uint32_t **data, unsigned *width, unsigned *height) { - int zstatus; + bool zstatus; + enum trans_stream_error terror; + uint32_t rd, wn; struct rpng_process *process = (struct rpng_process*)rpng->process; - bool to_continue = (process->stream_backend->stream_get_avail_in(process->stream) > 0 - && process->stream_backend->stream_get_avail_out(process->stream) > 0); + bool to_continue = (process->avail_in > 0 + && process->avail_out > 0); if (!to_continue) goto end; - zstatus = process->stream_backend->stream_decompress_data_to_file_iterate(process->stream); + zstatus = process->stream_backend->trans(process->stream, false, &rd, &wn, &terror); - switch (zstatus) - { - case 1: - goto end; - case -1: - goto error; - default: - break; - } + if (!zstatus && terror != TRANS_STREAM_ERROR_BUFFER_FULL) + goto error; - return 0; + process->avail_in -= rd; + process->avail_out -= wn; + process->total_out += wn; + + if (terror) + return 0; end: process->stream_backend->stream_free(process->stream); + process->stream = NULL; *width = rpng->ihdr.width; *height = rpng->ihdr.height; @@ -870,7 +872,7 @@ static struct rpng_process *rpng_process_init(rpng_t *rpng, unsigned *width, uns if (!process) return NULL; - process->stream_backend = file_archive_get_zlib_file_backend(); + process->stream_backend = trans_stream_get_zlib_inflate_backend(); png_pass_geom(&rpng->ihdr, rpng->ihdr.width, rpng->ihdr.height, NULL, NULL, &process->inflate_buf_size); @@ -885,23 +887,22 @@ static struct rpng_process *rpng_process_init(rpng_t *rpng, unsigned *width, uns return NULL; } - if (!process->stream_backend->stream_decompress_init(process->stream)) - { - free(process); - return NULL; - } - inflate_buf = (uint8_t*)malloc(process->inflate_buf_size); if (!inflate_buf) goto error; process->inflate_buf = inflate_buf; - process->stream_backend->stream_set( + process->avail_in = rpng->idat_buf.size; + process->avail_out = process->inflate_buf_size; + process->total_out = 0; + process->stream_backend->set_in( process->stream, - rpng->idat_buf.size, - process->inflate_buf_size, rpng->idat_buf.data, - process->inflate_buf); + rpng->idat_buf.size); + process->stream_backend->set_out( + process->stream, + process->inflate_buf, + process->inflate_buf_size); return process; @@ -1130,7 +1131,8 @@ void rpng_free(rpng_t *rpng) { if (rpng->process->stream_backend) rpng->process->stream_backend->stream_free(rpng->process->stream); - free(rpng->process->stream); + else + free(rpng->process->stream); } free(rpng->process); } diff --git a/libretro-common/formats/png/rpng_encode.c b/libretro-common/formats/png/rpng_encode.c index 000e876a98..f580ed2646 100644 --- a/libretro-common/formats/png/rpng_encode.c +++ b/libretro-common/formats/png/rpng_encode.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include "rpng_internal.h" @@ -213,7 +213,7 @@ static bool rpng_save_image(const char *path, bool ret = true; struct png_ihdr ihdr = {0}; - const struct file_archive_file_backend *stream_backend = NULL; + const struct trans_stream_backend *stream_backend = NULL; size_t encode_buf_size = 0; uint8_t *encode_buf = NULL; uint8_t *deflate_buf = NULL; @@ -225,11 +225,13 @@ static bool rpng_save_image(const char *path, uint8_t *prev_encoded = NULL; uint8_t *encode_target = NULL; void *stream = NULL; + uint32_t total_in = 0; + uint32_t total_out = 0; RFILE *file = filestream_open(path, RFILE_MODE_WRITE, -1); if (!file) GOTO_END_ERROR(); - stream_backend = file_archive_get_zlib_file_backend(); + stream_backend = trans_stream_get_zlib_deflate_backend(); if (filestream_write(file, png_magic, sizeof(png_magic)) != sizeof(png_magic)) GOTO_END_ERROR(); @@ -328,26 +330,23 @@ static bool rpng_save_image(const char *path, if (!stream) GOTO_END_ERROR(); - stream_backend->stream_set( + stream_backend->set_in( stream, - encode_buf_size, - encode_buf_size * 2, encode_buf, - deflate_buf + 8); + encode_buf_size); + stream_backend->set_out( + stream, + deflate_buf + 8, + encode_buf_size * 2); - stream_backend->stream_compress_init(stream, 9); - - if (stream_backend->stream_compress_data_to_file(stream) != 1) + if (!stream_backend->trans(stream, true, &total_in, &total_out, NULL)) { - stream_backend->stream_compress_free(stream); GOTO_END_ERROR(); } - stream_backend->stream_compress_free(stream); - memcpy(deflate_buf + 4, "IDAT", 4); - dword_write_be(deflate_buf + 0, ((uint32_t)stream_backend->stream_get_total_out(stream))); - if (!png_write_idat(file, deflate_buf, ((size_t)stream_backend->stream_get_total_out(stream) + 8))) + dword_write_be(deflate_buf + 0, ((uint32_t)total_out)); + if (!png_write_idat(file, deflate_buf, ((size_t)total_out + 8))) GOTO_END_ERROR(); if (!png_write_iend(file)) diff --git a/libretro-common/include/file/archive_file.h b/libretro-common/include/file/archive_file.h index 048387c857..407bc3cfc7 100644 --- a/libretro-common/include/file/archive_file.h +++ b/libretro-common/include/file/archive_file.h @@ -121,19 +121,9 @@ struct file_archive_file_backend { void *(*stream_new)(void); void (*stream_free)(void *); - void (*stream_set)(void *, uint32_t, uint32_t, - const uint8_t *, uint8_t *); - uint32_t (*stream_get_avail_in)(void*); - uint32_t (*stream_get_avail_out)(void*); - uint64_t (*stream_get_total_out)(void*); - void (*stream_decrement_total_out)(void *, unsigned); - bool (*stream_decompress_init)(void *); bool (*stream_decompress_data_to_file_init)( file_archive_file_handle_t *, const uint8_t *, uint32_t, uint32_t); int (*stream_decompress_data_to_file_iterate)(void *); - void (*stream_compress_init)(void *, int); - void (*stream_compress_free)(void *); - int (*stream_compress_data_to_file)(void *); uint32_t (*stream_crc_calculate)(uint32_t, const uint8_t *, size_t); int (*compressed_file_read)(const char *path, const char *needle, void **buf, const char *optional_outfile); diff --git a/libretro-common/include/streams/trans_stream.h b/libretro-common/include/streams/trans_stream.h new file mode 100644 index 0000000000..6cb7420f19 --- /dev/null +++ b/libretro-common/include/streams/trans_stream.h @@ -0,0 +1,101 @@ +/* Copyright (C) 2016 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (trans_stream.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef LIBRETRO_SDK_TRANS_STREAM_H__ +#define LIBRETRO_SDK_TRANS_STREAM_H__ + +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include + +enum trans_stream_error +{ + TRANS_STREAM_ERROR_NONE = 0, + TRANS_STREAM_ERROR_AGAIN, /* more work to do */ + TRANS_STREAM_ERROR_ALLOCATION_FAILURE, /* malloc failure */ + TRANS_STREAM_ERROR_INVALID, /* invalid state */ + TRANS_STREAM_ERROR_BUFFER_FULL, /* output buffer full */ + TRANS_STREAM_ERROR_OTHER +}; + +struct trans_stream_backend +{ + const char *ident; + const struct trans_stream_backend *reverse; + + /* Create a stream data structure */ + void *(*stream_new)(void); + + /* Free it */ + void (*stream_free)(void *); + + /* (Optional) Set extra properties, defined per transcoder */ + bool (*define)(void *, const char *, uint32_t); + + /* Set our input source */ + void (*set_in)(void *, const uint8_t *, uint32_t); + + /* Set our output target */ + void (*set_out)(void *, uint8_t *, uint32_t); + + /* Perform a transcoding, flushing/finalizing if asked to. Writes out how + * many bytes were read and written. Error target optional. */ + bool (*trans)(void *, bool, uint32_t *, uint32_t *, enum trans_stream_error *); +}; + +/** + * trans_stream_trans_full: + * @backend : transcoding backend + * @data : (optional) existing stream data, or a target + * for the new stream data to be saved + * @in : input data + * @in_size : input size + * @out : output data + * @out_size : output size + * @error : (optional) output for error code + * + * Perform a full transcoding from a source to a destination. + */ +bool trans_stream_trans_full( + struct trans_stream_backend *backend, void **data, + const uint8_t *in, uint32_t in_size, + uint8_t *out, uint32_t out_size, + enum trans_stream_error *error); + +const struct trans_stream_backend* trans_stream_get_zlib_deflate_backend(void); +const struct trans_stream_backend* trans_stream_get_zlib_inflate_backend(void); +const struct trans_stream_backend* trans_stream_get_pipe_backend(void); + +extern const struct trans_stream_backend zlib_deflate_backend; +extern const struct trans_stream_backend zlib_inflate_backend; +extern const struct trans_stream_backend pipe_backend; + +#endif + diff --git a/libretro-common/streams/trans_stream.c b/libretro-common/streams/trans_stream.c new file mode 100644 index 0000000000..d6acfa5e00 --- /dev/null +++ b/libretro-common/streams/trans_stream.c @@ -0,0 +1,95 @@ +/* Copyright (C) 2016 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (trans_stream.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include + +/** + * trans_stream_trans_full: + * @data : (optional) existing stream data, or a target + * for the new stream data to be saved + * @in : input data + * @in_size : input size + * @out : output data + * @out_size : output size + * @error : (optional) output for error code + * + * Perform a full transcoding from a source to a destination. + */ +bool trans_stream_trans_full( + struct trans_stream_backend *backend, void **data, + const uint8_t *in, uint32_t in_size, + uint8_t *out, uint32_t out_size, + enum trans_stream_error *error) +{ + void *rdata; + bool ret; + uint32_t rd, wn; + + if (data && *data) + { + rdata = *data; + } + else + { + rdata = backend->stream_new(); + if (!rdata) + { + if (error) + *error = TRANS_STREAM_ERROR_ALLOCATION_FAILURE; + return false; + } + } + + backend->set_in(rdata, in, in_size); + backend->set_out(rdata, out, out_size); + ret = backend->trans(rdata, true, &rd, &wn, error); + + if (data) + *data = rdata; + else + backend->stream_free(rdata); + + return ret; +} + +const struct trans_stream_backend* trans_stream_get_zlib_deflate_backend(void) +{ +#if HAVE_ZLIB + return &zlib_deflate_backend; +#else + return NULL; +#endif +} + +const struct trans_stream_backend* trans_stream_get_zlib_inflate_backend(void) +{ +#if HAVE_ZLIB + return &zlib_inflate_backend; +#else + return NULL; +#endif +} + +const struct trans_stream_backend* trans_stream_get_pipe_backend(void) +{ + return &pipe_backend; +} diff --git a/libretro-common/streams/trans_stream_pipe.c b/libretro-common/streams/trans_stream_pipe.c new file mode 100644 index 0000000000..255e48bfb8 --- /dev/null +++ b/libretro-common/streams/trans_stream_pipe.c @@ -0,0 +1,95 @@ +/* Copyright (C) 2016 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (trans_stream_zlib.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include + +struct pipe_trans_stream +{ + const uint8_t *in; + uint8_t *out; + uint32_t in_size, out_size; +}; + +static void *pipe_stream_new(void) +{ + return (struct pipe_trans_stream*)calloc(1, sizeof(struct pipe_trans_stream)); +} + +static void pipe_stream_free(void *data) +{ + free(data); +} + +static void pipe_set_in(void *data, const uint8_t *in, uint32_t in_size) +{ + struct pipe_trans_stream *p = (struct pipe_trans_stream *) data; + p->in = in; + p->in_size = in_size; +} + +static void pipe_set_out(void *data, uint8_t *out, uint32_t out_size) +{ + struct pipe_trans_stream *p = (struct pipe_trans_stream *) data; + p->out = out; + p->out_size = out_size; +} + +static bool pipe_trans( + void *data, bool flush, + uint32_t *rd, uint32_t *wn, + enum trans_stream_error *error) +{ + struct pipe_trans_stream *p = (struct pipe_trans_stream *) data; + + if (p->out_size < p->in_size) + { + memcpy(p->out, p->in, p->out_size); + *rd = *wn = p->out_size; + p->in += p->out_size; + p->out += p->out_size; + *error = TRANS_STREAM_ERROR_BUFFER_FULL; + return false; + } + else + { + memcpy(p->out, p->in, p->in_size); + *rd = *wn = p->in_size; + p->in += p->in_size; + p->out += p->in_size; + *error = TRANS_STREAM_ERROR_NONE; + return true; + } +} + +const struct trans_stream_backend pipe_backend = { + "pipe", + &pipe_backend, + pipe_stream_new, + pipe_stream_free, + NULL, + pipe_set_in, + pipe_set_out, + pipe_trans +}; diff --git a/libretro-common/streams/trans_stream_zlib.c b/libretro-common/streams/trans_stream_zlib.c new file mode 100644 index 0000000000..dfa9cacba3 --- /dev/null +++ b/libretro-common/streams/trans_stream_zlib.c @@ -0,0 +1,265 @@ +/* Copyright (C) 2016 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (trans_stream_zlib.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include +#include + +struct zlib_trans_stream +{ + z_stream z; + int ex; /* window_bits or level */ + bool inited; +}; + +static void *zlib_deflate_stream_new(void) +{ + struct zlib_trans_stream *ret = (struct zlib_trans_stream*)calloc(1, sizeof(struct zlib_trans_stream)); + if (ret) + ret->ex = 9; + return (void *) ret; +} + +static void *zlib_inflate_stream_new(void) +{ + struct zlib_trans_stream *ret = (struct zlib_trans_stream*)calloc(1, sizeof(struct zlib_trans_stream)); + if (ret) + ret->ex = MAX_WBITS; + return (void *) ret; +} + +static void zlib_deflate_stream_free(void *data) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + if (z->inited) + deflateEnd(&z->z); + free(z); +} + +static void zlib_inflate_stream_free(void *data) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + if (z->inited) + inflateEnd(&z->z); + free(z); +} + +static bool zlib_deflate_define(void *data, const char *prop, uint32_t val) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + if (!strcmp(prop, "level")) + { + z->ex = (int) val; + return true; + } + return false; +} + +static bool zlib_inflate_define(void *data, const char *prop, uint32_t val) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + if (!strcmp(prop, "window_bits")) + { + z->ex = (int) val; + return true; + } + return false; +} + +static void zlib_deflate_set_in(void *data, const uint8_t *in, uint32_t in_size) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + z->z.next_in = (uint8_t *) in; + z->z.avail_in = in_size; + if (!z->inited) + { + deflateInit(&z->z, z->ex); + z->inited = true; + } +} + +static void zlib_inflate_set_in(void *data, const uint8_t *in, uint32_t in_size) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + z->z.next_in = (uint8_t *) in; + z->z.avail_in = in_size; + if (!z->inited) + { + inflateInit2(&z->z, z->ex); + z->inited = true; + } +} + +static void zlib_set_out(void *data, uint8_t *out, uint32_t out_size) +{ + struct zlib_trans_stream *z = (struct zlib_trans_stream *) data; + z->z.next_out = out; + z->z.avail_out = out_size; +} + +static bool zlib_deflate_trans( + void *data, bool flush, + uint32_t *rd, uint32_t *wn, + enum trans_stream_error *error) +{ + int zret; + bool ret; + uint32_t pre_avail_in, pre_avail_out; + struct zlib_trans_stream *zt = (struct zlib_trans_stream *) data; + z_stream *z = &zt->z; + + if (!zt->inited) + { + deflateInit(z, zt->ex); + zt->inited = true; + } + + pre_avail_in = z->avail_in; + pre_avail_out = z->avail_out; + zret = deflate(z, flush ? Z_FINISH : Z_NO_FLUSH); + + if (zret == Z_OK) + { + if (error) + *error = TRANS_STREAM_ERROR_AGAIN; + } + else if (zret == Z_STREAM_END) + { + if (error) + *error = TRANS_STREAM_ERROR_NONE; + } + else + { + if (error) + *error = TRANS_STREAM_ERROR_OTHER; + return false; + } + ret = true; + + if (z->avail_out == 0) + { + /* Filled buffer, maybe an error */ + if (z->avail_in != 0) + { + ret = false; + if (error) + *error = TRANS_STREAM_ERROR_BUFFER_FULL; + } + } + + *rd = pre_avail_in - z->avail_in; + *wn = pre_avail_out - z->avail_out; + + if (flush && zret == Z_STREAM_END) + { + deflateEnd(z); + zt->inited = false; + } + + return ret; +} + +static bool zlib_inflate_trans( + void *data, bool flush, + uint32_t *rd, uint32_t *wn, + enum trans_stream_error *error) +{ + int zret; + bool ret; + uint32_t pre_avail_in, pre_avail_out; + struct zlib_trans_stream *zt = (struct zlib_trans_stream *) data; + z_stream *z = &zt->z; + + if (!zt->inited) + { + inflateInit2(z, zt->ex); + zt->inited = true; + } + + pre_avail_in = z->avail_in; + pre_avail_out = z->avail_out; + zret = inflate(z, flush ? Z_FINISH : Z_NO_FLUSH); + + if (zret == Z_OK) + { + if (error) + *error = TRANS_STREAM_ERROR_AGAIN; + } + else if (zret == Z_STREAM_END) + { + if (error) + *error = TRANS_STREAM_ERROR_NONE; + } + else + { + if (error) + *error = TRANS_STREAM_ERROR_OTHER; + return false; + } + ret = true; + + if (z->avail_out == 0) + { + /* Filled buffer, maybe an error */ + if (z->avail_in != 0) + { + ret = false; + if (error) + *error = TRANS_STREAM_ERROR_BUFFER_FULL; + } + } + + *rd = pre_avail_in - z->avail_in; + *wn = pre_avail_out - z->avail_out; + + if (flush && zret == Z_STREAM_END) + { + inflateEnd(z); + zt->inited = false; + } + + return ret; +} + +const struct trans_stream_backend zlib_deflate_backend = { + "zlib_deflate", + &zlib_inflate_backend, + zlib_deflate_stream_new, + zlib_deflate_stream_free, + zlib_deflate_define, + zlib_deflate_set_in, + zlib_set_out, + zlib_deflate_trans +}; + +const struct trans_stream_backend zlib_inflate_backend = { + "zlib_inflate", + &zlib_deflate_backend, + zlib_inflate_stream_new, + zlib_inflate_stream_free, + zlib_inflate_define, + zlib_inflate_set_in, + zlib_set_out, + zlib_inflate_trans +}; diff --git a/network/netplay/README b/network/netplay/README index 8f0ca71057..e5a5384163 100644 --- a/network/netplay/README +++ b/network/netplay/README @@ -60,10 +60,10 @@ During the frame of execution, when the core requests input, it receives the input from the state buffer, both local and real or simulated remote. Post-frame, it checks whether it's read more than it's actioned, i.e. if read > -other self > other. If so, it first checks whether its simulated remote data -was correct. If it was, it simply moves other up. If not, it rewinds to other -(by loading the serialized state there) and runs the core in replay mode with -the real data up to the least of self and read, then sets other to that. +other and self > other. If so, it first checks whether its simulated remote +data was correct. If it was, it simply moves other up. If not, it rewinds to +other (by loading the serialized state there) and runs the core in replay mode +with the real data up to the least of self and read, then sets other to that. When in Netplay mode, the callback for receiving input is replaced by input_state_net. It is the role of input_state_net to combine the true local @@ -146,8 +146,18 @@ Command: LOAD_SAVESTATE Payload: { frame number: uint32 + uncompressed size: uint32 serialized save state: blob (variable size) } Description: Cause the other side to load a savestate, notionally one which the sending - side has also loaded. + side has also loaded. If both sides support zlib compression, the + serialized state is zlib compressed. Otherwise it is uncompressed. + +Command: PAUSE +Payload: None + Indicates that the core is paused. The receiving peer should also pause. + +Command: RESUME +Payload: None + Indicates that the core is no longer paused. diff --git a/network/netplay/netplay.c b/network/netplay/netplay.c index 14fcd618a9..8bc5bc5907 100644 --- a/network/netplay/netplay.c +++ b/network/netplay/netplay.c @@ -584,6 +584,8 @@ static bool netplay_get_cmd(netplay_t *netplay) case NETPLAY_CMD_LOAD_SAVESTATE: { uint32_t frame; + uint32_t isize; + uint32_t rd, wn; /* Make sure we're ready for it */ if (netplay->quirks & NETPLAY_QUIRK_INITIALIZATION) @@ -612,9 +614,9 @@ static bool netplay_get_cmd(netplay_t *netplay) * (strangely) force a rewind to the frame we're already on, so it * gets loaded. This is just to avoid having reloading implemented in * too many places. */ - if (cmd_size > netplay->state_size + sizeof(uint32_t)) + if (cmd_size > netplay->zbuffer_size + 2*sizeof(uint32_t)) { - RARCH_ERR("CMD_LOAD_SAVESTATE received an unexpected save state size.\n"); + RARCH_ERR("CMD_LOAD_SAVESTATE received an unexpected payload size.\n"); return netplay_cmd_nak(netplay); } @@ -631,14 +633,34 @@ static bool netplay_get_cmd(netplay_t *netplay) return netplay_cmd_nak(netplay); } + if (!socket_receive_all_blocking(netplay->fd, &isize, sizeof(isize))) + { + RARCH_ERR("CMD_LOAD_SAVESTATE failed to receive inflated size.\n"); + return netplay_cmd_nak(netplay); + } + isize = ntohl(isize); + + if (isize != netplay->state_size) + { + RARCH_ERR("CMD_LOAD_SAVESTATE received an unexpected save state size.\n"); + return netplay_cmd_nak(netplay); + } + if (!socket_receive_all_blocking(netplay->fd, - netplay->buffer[netplay->read_ptr].state, - cmd_size - sizeof(uint32_t))) + netplay->zbuffer, cmd_size - 2*sizeof(uint32_t))) { RARCH_ERR("CMD_LOAD_SAVESTATE failed to receive savestate.\n"); return netplay_cmd_nak(netplay); } + /* And decompress it */ + netplay->decompression_backend->set_in(netplay->decompression_stream, + netplay->zbuffer, cmd_size - 2*sizeof(uint32_t)); + netplay->decompression_backend->set_out(netplay->decompression_stream, + netplay->buffer[netplay->read_ptr].state, netplay->state_size); + netplay->decompression_backend->trans(netplay->decompression_stream, + true, &rd, &wn, NULL); + /* Skip ahead if it's past where we are */ if (frame > netplay->self_frame_count) { @@ -1101,6 +1123,15 @@ bool netplay_init_serialization(netplay_t *netplay) } } + netplay->zbuffer_size = netplay->state_size * 2; + netplay->zbuffer = (uint8_t *) calloc(netplay->zbuffer_size, 1); + if (!netplay->zbuffer) + { + netplay->quirks |= NETPLAY_QUIRK_NO_TRANSMISSION; + netplay->zbuffer_size = 0; + return false; + } + return true; } @@ -1292,7 +1323,8 @@ void netplay_free(netplay_t *netplay) free(netplay->spectate.input); } - else + + if (netplay->buffer) { for (i = 0; i < netplay->buffer_size; i++) if (netplay->buffer[i].state) @@ -1301,6 +1333,12 @@ void netplay_free(netplay_t *netplay) free(netplay->buffer); } + if (netplay->zbuffer) + free(netplay->zbuffer); + + if (netplay->compression_stream) + netplay->compression_backend->stream_free(netplay->compression_stream); + if (netplay->addr) freeaddrinfo_retro(netplay->addr); @@ -1391,8 +1429,9 @@ void netplay_frontend_paused(netplay_t *netplay, bool paused) void netplay_load_savestate(netplay_t *netplay, retro_ctx_serialize_info_t *serial_info, bool save) { - uint32_t header[3]; + uint32_t header[4]; retro_ctx_serialize_info_t tmp_serial_info; + uint32_t rd, wn; if (!netplay->has_connection) return; @@ -1442,10 +1481,23 @@ void netplay_load_savestate(netplay_t *netplay, | NETPLAY_QUIRK_NO_TRANSMISSION)) return; - /* And send it to the peer (FIXME: this is an ugly way to do this) */ + /* Compress it */ + netplay->compression_backend->set_in(netplay->compression_stream, + serial_info->data_const, serial_info->size); + netplay->compression_backend->set_out(netplay->compression_stream, + netplay->zbuffer, netplay->zbuffer_size); + if (!netplay->compression_backend->trans(netplay->compression_stream, + true, &rd, &wn, NULL)) + { + hangup(netplay); + return; + } + + /* And send it to the peer */ header[0] = htonl(NETPLAY_CMD_LOAD_SAVESTATE); - header[1] = htonl(serial_info->size + sizeof(uint32_t)); + header[1] = htonl(wn + 2*sizeof(uint32_t)); header[2] = htonl(netplay->self_frame_count); + header[3] = htonl(serial_info->size); if (!socket_send_all_blocking(netplay->fd, header, sizeof(header), false)) { @@ -1454,7 +1506,7 @@ void netplay_load_savestate(netplay_t *netplay, } if (!socket_send_all_blocking(netplay->fd, - serial_info->data_const, serial_info->size, false)) + netplay->zbuffer, wn, false)) { hangup(netplay); return; diff --git a/network/netplay/netplay_common.c b/network/netplay/netplay_common.c index da7a9a9926..f3edd0ab14 100644 --- a/network/netplay/netplay_common.c +++ b/network/netplay/netplay_common.c @@ -166,8 +166,9 @@ bool netplay_handshake(netplay_t *netplay) char msg[512]; uint32_t *content_crc_ptr = NULL; void *sram = NULL; - uint32_t header[4] = {0}; + uint32_t header[5] = {0}; bool is_server = netplay->is_server; + int compression = 0; msg[0] = '\0'; @@ -182,6 +183,7 @@ bool netplay_handshake(netplay_t *netplay) header[1] = htonl(netplay_impl_magic()); header[2] = htonl(mem_info.size); header[3] = htonl(local_pmagic); + header[4] = htonl(NETPLAY_COMPRESSION_SUPPORTED); if (!socket_send_all_blocking(netplay->fd, header, sizeof(header), false)) return false; @@ -230,6 +232,36 @@ bool netplay_handshake(netplay_t *netplay) return false; } + /* Clear any existing compression */ + if (netplay->compression_stream) + netplay->compression_backend->stream_free(netplay->compression_stream); + if (netplay->decompression_stream) + netplay->decompression_backend->stream_free(netplay->decompression_stream); + + /* Check what compression is supported */ + compression = ntohl(header[4]); + compression &= NETPLAY_COMPRESSION_SUPPORTED; + if (compression & NETPLAY_COMPRESSION_ZLIB) + { + netplay->compression_backend = trans_stream_get_zlib_deflate_backend(); + if (!netplay->compression_backend) + netplay->compression_backend = trans_stream_get_pipe_backend(); + } + else + { + netplay->compression_backend = trans_stream_get_pipe_backend(); + } + netplay->decompression_backend = netplay->compression_backend->reverse; + + /* Allocate our compression stream */ + netplay->compression_stream = netplay->compression_backend->stream_new(); + netplay->decompression_stream = netplay->decompression_backend->stream_new(); + if (!netplay->compression_stream || !netplay->decompression_stream) + { + RARCH_ERR("Failed to allocate compression transcoder!\n"); + return false; + } + /* Client sends nickname first, server replies with nickname */ if (!is_server) { diff --git a/network/netplay/netplay_private.h b/network/netplay/netplay_private.h index 4cfbe9273e..40689561c0 100644 --- a/network/netplay/netplay_private.h +++ b/network/netplay/netplay_private.h @@ -21,6 +21,7 @@ #include #include +#include #include #include "../../core.h" @@ -35,7 +36,7 @@ #define MAX_SPECTATORS 16 #define RARCH_DEFAULT_PORT 55435 -#define NETPLAY_PROTOCOL_VERSION 2 +#define NETPLAY_PROTOCOL_VERSION 3 #define PREV_PTR(x) ((x) == 0 ? netplay->buffer_size - 1 : (x) - 1) #define NEXT_PTR(x) ((x + 1) % netplay->buffer_size) @@ -66,6 +67,14 @@ #define NETPLAY_QUIRK_MAP_PLATFORM_DEPENDENT \ (RETRO_SERIALIZATION_QUIRK_PLATFORM_DEPENDENT) +/* Compression protocols supported */ +#define NETPLAY_COMPRESSION_ZLIB (1<<0) +#if HAVE_ZLIB +#define NETPLAY_COMPRESSION_SUPPORTED NETPLAY_COMPRESSION_ZLIB +#else +#define NETPLAY_COMPRESSION_SUPPORTED 0 +#endif + struct delta_frame { bool used; /* a bit derpy, but this is how we know if the delta's been used at all */ @@ -122,6 +131,16 @@ struct netplay struct delta_frame *buffer; size_t buffer_size; + /* Compression transcoder */ + const struct trans_stream_backend *compression_backend; + void *compression_stream; + const struct trans_stream_backend *decompression_backend; + void *decompression_stream; + + /* A buffer into which to compress frames for transfer */ + uint8_t *zbuffer; + size_t zbuffer_size; + /* Pointer where we are now. */ size_t self_ptr; /* Points to the last reliable state that self ever had. */