diff --git a/deps/xdelta3/xdelta3-decode.h b/deps/xdelta3/xdelta3-decode.h index 37fb2f7508..55822252c4 100644 --- a/deps/xdelta3/xdelta3-decode.h +++ b/deps/xdelta3/xdelta3-decode.h @@ -162,6 +162,9 @@ xd3_decode_allocate (xd3_stream *stream, uint8_t **buf_ptr, usize_t *buf_alloc) { + IF_DEBUG2 (DP(RINT "[xd3_decode_allocate] size %"W"u alloc %"W"u\n", + size, *buf_alloc)); + if (*buf_ptr != NULL && *buf_alloc < size) { xd3_free (stream, *buf_ptr); @@ -187,6 +190,9 @@ xd3_decode_section (xd3_stream *stream, xd3_decode_state nstate, int copy) { + XD3_ASSERT (section->pos <= section->size); + XD3_ASSERT (stream->dec_state != nstate); + if (section->pos < section->size) { usize_t sect_take; @@ -201,6 +207,8 @@ xd3_decode_section (xd3_stream *stream, /* No allocation/copy needed */ section->buf = stream->next_in; sect_take = section->size; + IF_DEBUG1 (DP(RINT "[xd3_decode_section] zerocopy %"W"u @ %"W"u avail %"W"u\n", + sect_take, section->pos, stream->avail_in)); } else { @@ -217,11 +225,17 @@ xd3_decode_section (xd3_stream *stream, section->size, & section->copied1, & section->alloc1))) + { return ret; + } section->buf = section->copied1; } + IF_DEBUG2 (DP(RINT "[xd3_decode_section] take %"W"u @ %"W"u [need %"W"u] avail %"W"u\n", + sect_take, section->pos, sect_need, stream->avail_in)); + XD3_ASSERT (section->pos + sect_take <= section->alloc1); + memcpy (section->copied1 + section->pos, stream->next_in, sect_take); @@ -236,10 +250,14 @@ xd3_decode_section (xd3_stream *stream, if (section->pos < section->size) { + IF_DEBUG1 (DP(RINT "[xd3_decode_section] further input required %"W"u\n", + section->size - section->pos)); stream->msg = "further input required"; return XD3_INPUT; } + XD3_ASSERT (section->pos == section->size); + stream->dec_state = nstate; section->buf_max = section->buf + section->size; section->pos = 0; @@ -269,13 +287,27 @@ xd3_decode_parse_halfinst (xd3_stream *stream, xd3_hinst *inst) /* For copy instructions, read address. */ if (inst->type >= XD3_CPY) { + IF_DEBUG2 ({ + static int cnt = 0; + XPR(NT "DECODE:%u: COPY at %"Q"u (winoffset %"W"u) " + "size %"W"u winaddr %"W"u\n", + cnt++, + stream->total_out + (stream->dec_position - + stream->dec_cpylen), + (stream->dec_position - stream->dec_cpylen), + inst->size, + inst->addr); + }); + if ((ret = xd3_decode_address (stream, stream->dec_position, inst->type - XD3_CPY, & stream->addr_sect.buf, stream->addr_sect.buf_max, & inst->addr))) + { return ret; + } /* Cannot copy an address before it is filled-in. */ if (inst->addr >= stream->dec_position) @@ -293,6 +325,30 @@ xd3_decode_parse_halfinst (xd3_stream *stream, xd3_hinst *inst) return XD3_INVALID_INPUT; } } + else + { + IF_DEBUG2 ({ + if (inst->type == XD3_ADD) + { + static int cnt; + XPR(NT "DECODE:%d: ADD at %"Q"u (winoffset %"W"u) size %"W"u\n", + cnt++, + (stream->total_out + stream->dec_position - stream->dec_cpylen), + stream->dec_position - stream->dec_cpylen, + inst->size); + } + else + { + static int cnt; + XD3_ASSERT (inst->type == XD3_RUN); + XPR(NT "DECODE:%d: RUN at %"Q"u (winoffset %"W"u) size %"W"u\n", + cnt++, + stream->total_out + stream->dec_position - stream->dec_cpylen, + stream->dec_position - stream->dec_cpylen, + inst->size); + } + }); + } /* Check: The instruction will not overflow the output buffer. */ if (stream->dec_position + inst->size > stream->dec_maxpos) @@ -360,6 +416,8 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) return XD3_INVALID_INPUT; } + XD3_ASSERT (inst->type != XD3_NOOP); + switch (inst->type) { case XD3_RUN: @@ -449,6 +507,7 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) int ret; xd3_blksize_add (&block, &blkoff, source, inst->addr); + XD3_ASSERT (blkoff < blksize); if ((ret = xd3_getblk (stream, block))) { @@ -468,10 +527,18 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) if ((source->onblk != blksize) && (blkoff + take > source->onblk)) { + IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q"u onblk " + "%"W"u blksize %"W"u blkoff %"W"u take %"W"u\n", + block, + source->onblk, + blksize, + blkoff, + take)); stream->msg = "source file too short"; return XD3_INVALID_INPUT; } + XD3_ASSERT (blkoff != blksize); /* Check if we have enough data on this block to * finish the instruction. */ @@ -485,6 +552,9 @@ xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) take = blksize - blkoff; inst->size -= take; inst->addr += take; + + /* because (blkoff + take > blksize), above */ + XD3_ASSERT (inst->size != 0); } } } @@ -592,6 +662,9 @@ xd3_decode_sections (xd3_stream *stream) } need += stream->data_sect.size; + /* The window may be entirely processed. */ + XD3_ASSERT (stream->dec_winbytes <= need); + /* Compute how much more input is needed. */ more = (need - stream->dec_winbytes); @@ -636,6 +709,8 @@ xd3_decode_sections (xd3_stream *stream) DEC_EMIT, copy))) { return ret; } } + XD3_ASSERT (stream->dec_winbytes == need); + if ((ret = xd3_decode_secondary_sections (stream))) { return ret; } if (stream->flags & XD3_SKIP_EMIT) @@ -656,6 +731,19 @@ xd3_decode_emit (xd3_stream *stream) { int ret; + /* Produce output: originally structured to allow reentrant code + * that fills as much of the output buffer as possible, but VCDIFF + * semantics allows to copy from anywhere from the target window, so + * instead allocate a sufficiently sized buffer after the target + * window length is decoded. + * + * This code still needs to be reentrant to allow XD3_GETSRCBLK to + * return control. This is handled by setting the + * stream->dec_currentN instruction types to XD3_NOOP after they + * have been processed. */ + XD3_ASSERT (! (stream->flags & XD3_SKIP_EMIT)); + XD3_ASSERT (stream->dec_tgtlen <= stream->space_out); + while (stream->inst_sect.buf != stream->inst_sect.buf_max || stream->dec_current1.type != XD3_NOOP || stream->dec_current2.type != XD3_NOOP) @@ -685,6 +773,8 @@ xd3_decode_emit (xd3_stream *stream) if (stream->avail_out != stream->dec_tgtlen) { + IF_DEBUG2 (DP(RINT "AVAIL_OUT(%"W"u) != DEC_TGTLEN(%"W"u)\n", + stream->avail_out, stream->dec_tgtlen)); stream->msg = "wrong window length"; return XD3_INVALID_INPUT; } @@ -919,6 +1009,9 @@ xd3_decode_input (xd3_stream *stream) if ((ret = xd3_decode_init_window (stream))) { return ret; } stream->dec_state = DEC_CPYLEN; + + IF_DEBUG2 (DP(RINT "--------- TARGET WINDOW %"Q"u -----------\n", + stream->current_window)); } case DEC_CPYLEN: @@ -1072,6 +1165,16 @@ xd3_decode_input (xd3_stream *stream) xd3_blksize_div(stream->dec_cpyoff, src, &src->cpyoff_blocks, &src->cpyoff_blkoff); + + IF_DEBUG2(DP(RINT + "[decode_cpyoff] %"Q"u " + "cpyblkno %"Q"u " + "cpyblkoff %"W"u " + "blksize %"W"u\n", + stream->dec_cpyoff, + src->cpyoff_blocks, + src->cpyoff_blkoff, + src->blksize)); } /* xd3_decode_emit returns XD3_OUTPUT on every success. */ @@ -1116,4 +1219,4 @@ xd3_decode_input (xd3_stream *stream) } } -#endif /* _XDELTA3_DECODE_H_*/ +#endif /* _XDELTA3_DECODE_H_*/ \ No newline at end of file diff --git a/deps/xdelta3/xdelta3-djw.h b/deps/xdelta3/xdelta3-djw.h index 411a054a79..a5750d15fc 100644 --- a/deps/xdelta3/xdelta3-djw.h +++ b/deps/xdelta3/xdelta3-djw.h @@ -305,6 +305,21 @@ heap_extract (usize_t *heap, const djw_heapen *ents, usize_t heap_last) return (djw_heapen*) & ents[smallest]; } +#if XD3_DEBUG +static void +heap_check (usize_t *heap, djw_heapen *ents, usize_t heap_last) +{ + usize_t i; + for (i = 1; i <= heap_last; i += 1) + { + /* Heap property: child not less than parent */ + XD3_ASSERT (! heap_less (& ents[heap[i]], & ents[heap[i/2]])); + + IF_DEBUG2 (DP(RINT "heap[%"W"u] = %u\n", i, ents[heap[i]].freq)); + } +} +#endif + /*********************************************************************/ /* MTF, 1/2 */ /*********************************************************************/ @@ -380,9 +395,15 @@ djw_build_prefix (const djw_weight *freq, uint8_t *clen, usize_t asize, usize_t usize_t total_bits; usize_t i; + IF_DEBUG (usize_t first_bits = 0); + /* Insert real symbol frequences. */ for (i = 0; i < asize; i += 1) + { ents[i+1].freq = freq[i]; + IF_DEBUG2 (DP(RINT "ents[%"W"i] = freq[%"W"u] = %d\n", + i+1, i, freq[i])); + } again: @@ -410,6 +431,11 @@ djw_build_prefix (const djw_weight *freq, uint8_t *clen, usize_t asize, usize_t } } + IF_DEBUG (heap_check (heap, ents, heap_last)); + + /* Must be at least one symbol, or else we can't get here. */ + XD3_ASSERT (heap_last != 0); + /* If there is only one symbol, fake a second to prevent zero-length * codes. */ if (heap_last == 1) @@ -435,6 +461,8 @@ djw_build_prefix (const djw_weight *freq, uint8_t *clen, usize_t asize, usize_t heap_insert (heap, ents, ++heap_last, ents_size++); } + IF_DEBUG (heap_check (heap, ents, heap_last)); + /* Now compute prefix code lengths, counting parents. */ for (i = 1; i < asize+1; i += 1) { @@ -452,11 +480,21 @@ djw_build_prefix (const djw_weight *freq, uint8_t *clen, usize_t asize, usize_t } /* clen is 0-origin, unlike ents. */ + IF_DEBUG2 (DP(RINT "clen[%"W"u] = %"W"u\n", i-1, b)); clen[i-1] = b; } + IF_DEBUG (if (first_bits == 0) first_bits = total_bits); + if (! overflow) + { + IF_DEBUG2 (if (first_bits != total_bits) + { + DP(RINT "code length overflow changed %"W"u bits\n", + total_bits - first_bits); + }); return total_bits; + } /* OPT: There is a non-looping way to fix overflow shown in zlib, but this * is easier (for now), as done in bzip2. */ @@ -487,6 +525,8 @@ djw_build_codes (usize_t *codes, const uint8_t *clen, usize_t asize, usize_t abs max_clen = xd3_max (max_clen, (usize_t) clen[i]); } + XD3_ASSERT (max_clen <= abs_max); + /* Generate a code for each symbol with the appropriate length. */ for (l = min_clen; l <= max_clen; l += 1) { @@ -500,6 +540,13 @@ djw_build_codes (usize_t *codes, const uint8_t *clen, usize_t asize, usize_t abs code <<= 1; } + + IF_DEBUG2 ({ + for (i = 0; i < asize; i += 1) + { + DP(RINT "code[%"W"u] = %"W"u\n", i, codes[i]); + } + }); } /*********************************************************************/ @@ -528,6 +575,8 @@ djw_compute_mtf_1_2 (djw_prefix *prefix, for (j = 0; mtf[j] != sym; j += 1) { } + XD3_ASSERT (j <= nsym); + for (k = j; k >= 1; k -= 1) { mtf[k] = mtf[k-1]; } mtf[0] = sym; @@ -579,6 +628,14 @@ djw_count_freqs (djw_weight *freq, xd3_output *input) while (++p < p_max); } + IF_DEBUG2 ({int i; + DP(RINT "freqs: "); + for (i = 0; i < ALPHABET_SIZE; i += 1) + { + DP(RINT "%u ", freq[i]); + } + DP(RINT "\n");}); + return size; } @@ -643,6 +700,7 @@ djw_encode_prefix (xd3_stream *stream, { num_to_encode -= 1; } + XD3_ASSERT (num_to_encode - DJW_EXTRA_12OFFSET < (1 << DJW_EXTRA_CODE_BITS)); /* Encode: # of extra codes */ if ((ret = xd3_encode_bits (stream, output, bstate, DJW_EXTRA_CODE_BITS, @@ -789,6 +847,11 @@ xd3_encode_howmany_groups (xd3_stream *stream, (*ret_groups) = cfg_groups; (*ret_sector_size) = cfg_sector_size; + XD3_ASSERT (cfg_groups > 0 && cfg_groups <= DJW_MAX_GROUPS); + XD3_ASSERT (cfg_groups == 1 || + (cfg_sector_size >= DJW_SECTORSZ_MULT && + cfg_sector_size <= DJW_SECTORSZ_MAX)); + return 0; } @@ -814,6 +877,8 @@ xd3_encode_huff (xd3_stream *stream, input_bytes = djw_count_freqs (real_freq, input); input_bits = input_bytes * 8; + XD3_ASSERT (input_bytes > 0); + if ((ret = xd3_encode_howmany_groups (stream, cfg, input_bytes, & groups, & sector_size))) { @@ -877,12 +942,18 @@ xd3_encode_huff (xd3_stream *stream, usize_t sym = *p++; usize_t bits = clen[sym]; + IF_DEBUG (output_bits -= bits); + if ((ret = xd3_encode_bits (stream, & output, & bstate, bits, code[sym]))) + { goto failure; + } } while (p < p_max); } + + XD3_ASSERT (output_bits == 0); } else { @@ -902,6 +973,7 @@ xd3_encode_huff (xd3_stream *stream, usize_t gbest_no; usize_t gpcnt; const uint8_t *p; + IF_DEBUG2 (usize_t gcount[DJW_MAX_GROUPS]); /* Encode: sector size (5 bits) */ if ((ret = xd3_encode_bits (stream, & output, & bstate, @@ -938,19 +1010,32 @@ xd3_encode_huff (xd3_stream *stream, djw_weight sum = 0; djw_weight goal = left / (groups - gp); + IF_DEBUG2 (usize_t nz = 0); + /* Due to the single-code granularity of this distribution, it may * be that we can't generate a distribution for each group. In that * case subtract one group and try again. If (inefficient), we're * testing group behavior, so don't mess things up. */ if (goal == 0 && !cfg->inefficient) { + IF_DEBUG2 (DP(RINT "too many groups (%"W"u), dropping one\n", + groups)); groups -= 1; goto regroup; } /* Sum == goal is possible when (cfg->inefficient)... */ while (sum < goal) + { + XD3_ASSERT (sym2 < ALPHABET_SIZE); + IF_DEBUG2 (nz += real_freq[sym2] != 0); sum += real_freq[sym2++]; + } + + IF_DEBUG2(DP(RINT "group %"W"u has symbols %"W"u..%"W"u (%"W"u non-zero) " + "(%u/%"W"u = %.3f)\n", + gp, sym1, sym2, nz, sum, + input_bytes, sum / (double)input_bytes);); for (s = 0; s < ALPHABET_SIZE; s += 1) { @@ -966,6 +1051,7 @@ xd3_encode_huff (xd3_stream *stream, niter += 1; gbest_no = 0; memset (evolve_freq, 0, sizeof (evolve_freq[0]) * groups); + IF_DEBUG2 (memset (gcount, 0, sizeof (gcount[0]) * groups)); /* For each input page (loop is irregular to allow non-pow2-size group * size. */ @@ -1015,7 +1101,9 @@ xd3_encode_huff (xd3_stream *stream, } } + XD3_ASSERT(gbest_no < gbest_max); gbest[gbest_no++] = winner; + IF_DEBUG2 (gcount[winner] += 1); p = p0; in = in0; @@ -1030,6 +1118,8 @@ xd3_encode_huff (xd3_stream *stream, } while (in != NULL); + XD3_ASSERT (gbest_no == gbest_max); + /* Recompute code lengths. */ output_bits = 0; for (gp = 0; gp < groups; gp += 1) @@ -1065,14 +1155,29 @@ xd3_encode_huff (xd3_stream *stream, * for the (output_bits==0) assert after all bits are output. */ if (any_zeros) { + IF_DEBUG2 (usize_t save_total = output_bits); + for (i = 0; i < ALPHABET_SIZE; i += 1) { if (evolve_zero[i]) { output_bits -= evolve_clen[gp][i]; } } + + IF_DEBUG2 (DP(RINT "evolve_zero reduced %"W"u bits in group %"W"u\n", + save_total - output_bits, gp)); } } + IF_DEBUG2( + DP(RINT "pass %"W"u total bits: %"W"u group uses: ", niter, output_bits); + for (gp = 0; gp < groups; gp += 1) { DP(RINT "%"W"u ", gcount[gp]); } + DP(RINT "\n"); + ); + /* End iteration. */ + + IF_DEBUG2 (if (niter > 1 && best_bits < output_bits) { + DP(RINT "iteration lost %"W"u bits\n", output_bits - best_bits); }); + if (niter == 1 || (niter < DJW_MAX_ITER && (best_bits - output_bits) >= DJW_MIN_IMPROVEMENT)) { @@ -1086,6 +1191,9 @@ xd3_encode_huff (xd3_stream *stream, goto nosecond; } + IF_DEBUG2 (DP(RINT "djw compression: %"W"u -> %0.3f\n", + input_bytes, output_bits / 8.0)); + /* Encode: prefix */ { uint8_t prefix_symbol[DJW_MAX_GROUPS * ALPHABET_SIZE]; @@ -1136,10 +1244,18 @@ xd3_encode_huff (xd3_stream *stream, usize_t gp_sel_bits = gbest_clen[gp_mtf]; usize_t gp_sel_code = gbest_code[gp_mtf]; + XD3_ASSERT (gp_mtf < groups+1); + if ((ret = xd3_encode_bits (stream, & output, & bstate, gp_sel_bits, gp_sel_code))) + { goto failure; + } + + IF_DEBUG (select_bits -= gp_sel_bits); } + + XD3_ASSERT (select_bits == 0); } /* Efficiency check. */ @@ -1172,6 +1288,8 @@ xd3_encode_huff (xd3_stream *stream, usize_t *gp_codes = evolve_code[gp_best]; uint8_t *gp_clens = evolve_clen[gp_best]; + XD3_ASSERT (sector < gbest_no); + sector += 1; /* Encode the sector data. */ @@ -1181,14 +1299,21 @@ xd3_encode_huff (xd3_stream *stream, usize_t bits = gp_clens[sym]; usize_t code = gp_codes[sym]; + IF_DEBUG (output_bits -= bits); + if ((ret = xd3_encode_bits (stream, & output, & bstate, bits, code))) + { goto failure; + } GP_PAGE (); } } while (in != NULL); + + XD3_ASSERT (select_bits == 0); + XD3_ASSERT (output_bits == 0); } } @@ -1231,6 +1356,9 @@ djw_build_decoder (xd3_stream *stream, usize_t min_clen; usize_t max_clen; + /* Assumption: the two temporary arrays are large enough to hold abs_max. */ + XD3_ASSERT (abs_max <= DJW_MAX_CODELEN); + /* This looks something like the start of zlib's inftrees.c */ memset (nr_clen, 0, sizeof (nr_clen[0]) * (abs_max+1)); @@ -1239,6 +1367,13 @@ djw_build_decoder (xd3_stream *stream, ci = clen; do { + /* Caller _must_ check that values are in-range. Most of the time the + * caller decodes a specific number of bits, which imply the max value, + * and the other time the caller decodes a huffman value, which must be + * in-range. Therefore, its an assertion and this function cannot + * otherwise fail. */ + XD3_ASSERT (*ci <= abs_max); + nr_clen[*ci++]++; } while (--i != 0); @@ -1333,6 +1468,7 @@ djw_decode_symbol (xd3_stream *stream, if (offset <= max_sym) { + IF_DEBUG2 (DP(RINT "(j) %"W"u ", code)); *sym = inorder[offset]; return 0; } @@ -1384,6 +1520,9 @@ djw_decode_clclen (xd3_stream *stream, /* Set the rest to zero. */ for (; i < DJW_TOTAL_CODES; i += 1) { cl_clen[i] = 0; } + /* No need to check for in-range clen values, because: */ + XD3_ASSERT (1 << DJW_CLCLEN_BITS == DJW_MAX_CLCLEN + 1); + /* Build the code-length decoder. */ djw_build_decoder (stream, DJW_TOTAL_CODES, DJW_MAX_CLCLEN, cl_clen, cl_inorder, cl_base, @@ -1645,6 +1784,8 @@ xd3_decode_huff (xd3_stream *stream, { gp = sel_group[c]; + XD3_ASSERT (gp < groups); + gp_inorder = inorder[gp]; gp_base = base[gp]; gp_limit = limit[gp]; @@ -1682,6 +1823,10 @@ xd3_decode_huff (xd3_stream *stream, } } + IF_REGRESSION (if ((ret = xd3_test_clean_bits (stream, & bstate))) + { goto fail; }); + XD3_ASSERT (ret == 0); + fail: xd3_free (stream, sel_group); diff --git a/deps/xdelta3/xdelta3-fgk.h b/deps/xdelta3/xdelta3-fgk.h index c583d194c2..027645d3a3 100644 --- a/deps/xdelta3/xdelta3-fgk.h +++ b/deps/xdelta3/xdelta3-fgk.h @@ -204,8 +204,12 @@ static int fgk_init (xd3_stream *stream, fgk_stream *h, int is_encode) fgk_factor_remaining(h); /* set ZFE and ZFR */ fgk_factor_remaining(h); /* set ZFDB according to prev state */ + IF_DEBUG (memset (h->alphabet, 0, sizeof (h->alphabet[0]) * h->total_nodes)); + for (ui = 0; ui < h->total_blocks-1; ui += 1) + { h->block_array[ui].block_freeptr = &h->block_array[ui + 1]; + } h->block_array[h->total_blocks - 1].block_freeptr = NULL; h->free_block = h->block_array; @@ -234,6 +238,8 @@ static usize_t fgk_encode_data (fgk_stream* h, usize_t n) { fgk_node *target_ptr = h->alphabet + n; + XD3_ASSERT (n < h->alphabet_size); + h->coded_depth = 0; /* First encode the binary representation of the nth remaining @@ -290,6 +296,8 @@ static usize_t fgk_encode_data (fgk_stream* h, usize_t n) */ static INLINE fgk_bit fgk_get_encoded_bit (fgk_stream *h) { + XD3_ASSERT (h->coded_depth > 0); + return h->coded_bits[--h->coded_depth]; } @@ -412,6 +420,9 @@ static void fgk_promote (fgk_stream *h, fgk_node *node) node->left_child && node->left_child->weight == 0) { + XD3_ASSERT (node->left_child == h->remaining_zeros); + XD3_ASSERT (node->right_child->weight == (node->weight+1)); /* child weight was already incremented */ + if (node->weight == (my_right->weight - 1) && my_right != h->root_node) { fgk_free_block (h, cur_block); @@ -613,6 +624,8 @@ static fgk_block* fgk_make_block (fgk_stream *h, fgk_node* lead) { fgk_block *ret = h->free_block; + XD3_ASSERT (h->free_block != NULL); + h->free_block = h->free_block->block_freeptr; ret->block_leader = lead; @@ -652,6 +665,8 @@ static void fgk_factor_remaining (fgk_stream *h) */ static INLINE int fgk_decode_bit (fgk_stream* h, fgk_bit b) { + XD3_ASSERT (b == 1 || b == 0); + if (IS_ADAPTIVE && h->decode_ptr->weight == 0) { usize_t bitsreq; @@ -827,6 +842,13 @@ xd3_decode_fgk (xd3_stream *stream, if (output == output_max) { + /* During regression testing: */ + IF_REGRESSION ({ + int ret; + bstate.cur_mask <<= 1; + if ((ret = xd3_test_clean_bits (stream, & bstate))) { return ret; } + }); + (*output_pos) = output; (*input_pos) = input; return 0; diff --git a/deps/xdelta3/xdelta3-hash.h b/deps/xdelta3/xdelta3-hash.h index fbf096d1b3..822aa0245f 100644 --- a/deps/xdelta3/xdelta3-hash.h +++ b/deps/xdelta3/xdelta3-hash.h @@ -20,6 +20,19 @@ #include "retro_inline.h" #include "xdelta3-internal.h" +#if XD3_DEBUG +#define SMALL_HASH_DEBUG1(s,inp) \ + uint32_t debug_state; \ + uint32_t debug_hval = xd3_checksum_hash (& (s)->small_hash, \ + xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look)) +#define SMALL_HASH_DEBUG2(s,inp) \ + XD3_ASSERT (debug_hval == xd3_checksum_hash (& (s)->small_hash, \ + xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look))) +#else +#define SMALL_HASH_DEBUG1(s,inp) +#define SMALL_HASH_DEBUG2(s,inp) +#endif /* XD3_DEBUG */ + #if UNALIGNED_OK #define UNALIGNED_READ32(dest,src) (*(dest)) = (*(uint32_t*)(src)) #else diff --git a/deps/xdelta3/xdelta3-internal.h b/deps/xdelta3/xdelta3-internal.h index 9c89620bd3..de7f09a54a 100644 --- a/deps/xdelta3/xdelta3-internal.h +++ b/deps/xdelta3/xdelta3-internal.h @@ -20,6 +20,31 @@ #include "retro_inline.h" #include "xdelta3.h" +typedef struct _main_file main_file; +typedef struct _main_extcomp main_extcomp; + +void main_buffree (void *ptr); +void* main_bufalloc (size_t size); +void main_file_init (main_file *xfile); +int main_file_close (main_file *xfile); +void main_file_cleanup (main_file *xfile); +int main_file_isopen (main_file *xfile); +int main_file_open (main_file *xfile, const char* name, int mode); +int main_file_exists (main_file *xfile); +int main_file_stat (main_file *xfile, xoff_t *size); +int xd3_whole_append_window (xd3_stream *stream); +int xd3_main_cmdline (int argc, char **argv); +int main_file_read (main_file *ifile, + uint8_t *buf, + size_t size, + size_t *nread, + const char *msg); +int main_file_write (main_file *ofile, uint8_t *buf, + usize_t size, const char *msg); +void* main_malloc (size_t size); +void main_free (void *ptr); + +int test_compare_files (const char* f0, const char* f1); usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno); xoff_t xd3_source_eof(const xd3_source *src); @@ -39,6 +64,7 @@ xd3_output* xd3_alloc_output (xd3_stream *stream, int xd3_encode_init_full (xd3_stream *stream); usize_t xd3_pow2_roundup (usize_t x); +long get_millisecs_now (void); int xd3_process_stream (int is_encode, xd3_stream *stream, int (*func) (xd3_stream *), @@ -49,6 +75,67 @@ int xd3_process_stream (int is_encode, usize_t *output_size, usize_t output_size_max); +#if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN +int xd3_main_cmdline (int argc, char **argv); +#endif + +#if REGRESSION_TEST +int xd3_selftest (void); +#endif + +/* main_file->mode values */ +typedef enum +{ + XO_READ = 0, + XO_WRITE = 1 +} main_file_modes; + +#ifndef XD3_POSIX +#define XD3_POSIX 0 +#endif +#ifndef XD3_STDIO +#define XD3_STDIO 0 +#endif +#ifndef XD3_WIN32 +#define XD3_WIN32 0 +#endif +#ifndef NOT_MAIN +#define NOT_MAIN 0 +#endif + +/* If none are set, default to posix. */ +#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0 +#undef XD3_POSIX +#define XD3_POSIX 1 +#endif + +struct _main_file +{ +#if XD3_WIN32 + HANDLE file; +#elif XD3_STDIO + FILE *file; +#elif XD3_POSIX + int file; +#endif + + int mode; /* XO_READ and XO_WRITE */ + const char *filename; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + char *filename_copy; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + const char *realname; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + const main_extcomp *compressor; /* External compression struct. */ + int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ + xoff_t nread; /* for input position */ + xoff_t nwrite; /* for output position */ + uint8_t *snprintf_buf; /* internal snprintf() use */ + int size_known; /* Set by main_set_souze */ + xoff_t source_position; /* for avoiding seek in getblk_func */ + int seek_failed; /* after seek fails once, try FIFO */ +}; + #ifndef UINT32_MAX #define UINT32_MAX 4294967295U #endif diff --git a/deps/xdelta3/xdelta3-second.h b/deps/xdelta3/xdelta3-second.h index 5401345d58..9d715a9fe8 100644 --- a/deps/xdelta3/xdelta3-second.h +++ b/deps/xdelta3/xdelta3-second.h @@ -65,10 +65,32 @@ static INLINE int xd3_decode_bits (xd3_stream *stream, done: + IF_DEBUG2 (DP(RINT "(d) %"W"u ", value)); + (*valuep) = value; return 0; } +#if REGRESSION_TEST +/* There may be extra bits at the end of secondary decompression, this macro + * checks for non-zero bits. This is overly strict, but helps pass the + * single-bit-error regression test. */ +static int +xd3_test_clean_bits (xd3_stream *stream, bit_state *bits) +{ + for (; bits->cur_mask != 0x100; bits->cur_mask <<= 1) + { + if (bits->cur_byte & bits->cur_mask) + { + stream->msg = "secondary decoder garbage"; + return XD3_INTERNAL; + } + } + + return 0; +} +#endif + static int xd3_get_secondary (xd3_stream *stream, xd3_sec_stream **sec_streamp, int is_encode) @@ -198,15 +220,23 @@ static INLINE int xd3_encode_bits (xd3_stream *stream, int ret; usize_t mask = 1 << nbits; + XD3_ASSERT (nbits > 0); + XD3_ASSERT (nbits < sizeof (usize_t) * 8); + XD3_ASSERT (value < mask); + do { mask >>= 1; if ((ret = xd3_encode_bit (stream, output, bits, value & mask))) + { return ret; + } } while (mask != 1); + IF_DEBUG2 (DP(RINT "(e) %"W"u ", value)); + return 0; } @@ -260,8 +290,18 @@ xd3_encode_secondary (xd3_stream *stream, comp_size += tmp_tail->next; } + XD3_ASSERT (comp_size == xd3_sizeof_output (tmp_head)); + XD3_ASSERT (tmp_tail != NULL); + if (comp_size < (orig_size - SECONDARY_MIN_SAVINGS) || cfg->inefficient) { + if (comp_size < orig_size) + { + IF_DEBUG1(DP(RINT "[encode_secondary] saved %"W"u bytes: %"W"u -> %"W"u (%0.2f%%)\n", + orig_size - comp_size, orig_size, comp_size, + 100.0 * (double) comp_size / (double) orig_size)); + } + xd3_free_output (stream, *head); *head = tmp_head; diff --git a/deps/xdelta3/xdelta3.c b/deps/xdelta3/xdelta3.c index a6b9b3d180..a6653a1c3b 100644 --- a/deps/xdelta3/xdelta3.c +++ b/deps/xdelta3/xdelta3.c @@ -443,6 +443,13 @@ XD3_MAKELIST(xd3_rlist, xd3_rinst, link); #define NEXTRUN(c) do { if ((c) == run_c) { run_l += 1; } \ else { run_c = (c); run_l = 1; } } while (0) +/* This CPP-conditional stuff can be cleaned up... */ +#if REGRESSION_TEST +#define IF_REGRESSION(x) x +#else +#define IF_REGRESSION(x) +#endif + /***********************************************************************/ #if XD3_ENCODER @@ -468,6 +475,8 @@ static int xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos); static int xd3_source_extend_match (xd3_stream *stream); static int xd3_srcwin_setup (xd3_stream *stream); static usize_t xd3_iopt_last_matched (xd3_stream *stream); +static int xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, + uint32_t num); static usize_t xd3_smatch (xd3_stream *stream, usize_t base, @@ -489,6 +498,19 @@ static void xd3_scksum_insert (xd3_stream *stream, usize_t pos); +#if XD3_DEBUG +static void xd3_verify_run_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_run_l, + uint8_t *x_run_c); +static void xd3_verify_large_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_cksum); +static void xd3_verify_small_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum); + +#endif /* XD3_DEBUG */ #endif /* XD3_ENCODER */ static int xd3_decode_allocate (xd3_stream *stream, usize_t size, @@ -869,6 +891,8 @@ xd3_build_code_table (const xd3_code_table_desc *desc, xd3_dinst *tbl) } } } + + XD3_ASSERT (d - tbl == 256); } /* This function generates the static default code table. */ @@ -919,6 +943,8 @@ xd3_choose_instruction (xd3_rinst *prev, xd3_rinst *inst) { uint8_t mode = inst->type - XD3_CPY; + XD3_ASSERT (inst->type >= XD3_CPY && inst->type < 12); + inst->code1 = 19 + 16 * mode; if (inst->size <= 18 && inst->size >= 4) @@ -935,14 +961,19 @@ xd3_choose_instruction (xd3_rinst *prev, xd3_rinst *inst) prev->code2 = (uint8_t)(163 + (mode * 12) + (3 * (prev->size - 1)) + (inst->size - 4)); + XD3_ASSERT (prev->code2 <= 234); } else if ( (inst->size == 4) && (mode >= 6) ) { prev->code2 = 235 + ((mode - 6) * 4) + (prev->size - 1); + + XD3_ASSERT (prev->code2 <= 246); } } } + + XD3_ASSERT (inst->code1 <= 162); } break; } @@ -1015,11 +1046,17 @@ xd3_round_blksize (usize_t sz, usize_t blksz) { usize_t mod = sz & (blksz-1); + XD3_ASSERT (xd3_check_pow2 (blksz, NULL) == 0); + if (mod == 0) + { return sz; + } if (sz > USIZE_T_MAXBLKSZ) + { return USIZE_T_MAXBLKSZ; + } return sz + (blksz - mod); } @@ -1248,6 +1285,8 @@ xd3_encode_address (xd3_stream *stream, bestd = addr; bestm = VCD_SELF; + XD3_ASSERT (addr < here); + SMALLEST_INT (bestd); if ((d = here-addr) < bestd) @@ -1373,8 +1412,16 @@ xd3_alloc (xd3_stream *stream, { void *a = stream->alloc (stream->opaque, elts, size); - if (!a) + if (a != NULL) + { + IF_DEBUG (stream->alloc_cnt += 1); + IF_DEBUG2 (DP(RINT "[stream %p malloc] size %"W"u ptr %p\n", + (void*)stream, elts * size, a)); + } + else + { stream->msg = "out of memory"; + } return a; } @@ -1384,7 +1431,13 @@ xd3_free (xd3_stream *stream, void *ptr) { if (ptr != NULL) + { + IF_DEBUG (stream->free_cnt += 1); + XD3_ASSERT (stream->free_cnt <= stream->alloc_cnt); + IF_DEBUG2 (DP(RINT "[stream %p free] %p\n", + (void*)stream, ptr)); stream->free (stream->opaque, ptr); + } } #if XD3_ENCODER @@ -1564,9 +1617,46 @@ xd3_free_stream (xd3_stream *stream) xd3_free (stream, stream->whole_target.inst); xd3_free (stream, stream->whole_target.wininfo); + XD3_ASSERT (stream->alloc_cnt == stream->free_cnt); + memset (stream, 0, sizeof (xd3_stream)); } +#if (XD3_DEBUG > 1 || VCDIFF_TOOLS) +static const char* +xd3_rtype_to_string (xd3_rtype type, int print_mode) +{ + switch (type) + { + case XD3_NOOP: + return "NOOP "; + case XD3_RUN: + return "RUN "; + case XD3_ADD: + return "ADD "; + default: break; + } + if (! print_mode) + { + return "CPY "; + } + switch (type) + { + case XD3_CPY + 0: return "CPY_0"; + case XD3_CPY + 1: return "CPY_1"; + case XD3_CPY + 2: return "CPY_2"; + case XD3_CPY + 3: return "CPY_3"; + case XD3_CPY + 4: return "CPY_4"; + case XD3_CPY + 5: return "CPY_5"; + case XD3_CPY + 6: return "CPY_6"; + case XD3_CPY + 7: return "CPY_7"; + case XD3_CPY + 8: return "CPY_8"; + case XD3_CPY + 9: return "CPY_9"; + default: return "CPY>9"; + } +} +#endif + /**************************************************************** Stream configuration ******************************************************************/ @@ -1776,24 +1866,47 @@ xd3_getblk (xd3_stream *stream, xoff_t blkno) if (stream->getblk == NULL) { + IF_DEBUG2 (DP(RINT "[getblk] XD3_GETSRCBLK %"Q"u\n", blkno)); stream->msg = "getblk source input"; return XD3_GETSRCBLK; } ret = stream->getblk (stream, source, blkno); if (ret != 0) + { + IF_DEBUG2 (DP(RINT "[getblk] app error blkno %"Q"u: %s\n", + blkno, xd3_strerror (ret))); return ret; + } + + IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q"u onblk " + "%"W"u blksize %"W"u max_blkno %"Q"u\n", blkno, source->onblk, + source->blksize, source->max_blkno)); } if (blkno > source->max_blkno) { source->max_blkno = blkno; - if (source->onblk == source->blksize) { } + if (source->onblk == source->blksize) + { + IF_DEBUG1 (DP(RINT "[getblk] full source blkno %"Q"u: " + "source length unknown %"Q"u\n", + blkno, + xd3_source_eof (source))); + } else if (!source->eof_known) + { + IF_DEBUG1 (DP(RINT "[getblk] eof block has %"W"u bytes; " + "source length known %"Q"u\n", + xd3_bytes_on_srcblk (source, blkno), + xd3_source_eof (source))); source->eof_known = 1; + } } + XD3_ASSERT (source->curblk != NULL); + if (blkno == source->max_blkno) { /* In case the application sets the source as 1 block w/ a @@ -1823,13 +1936,17 @@ xd3_set_source (xd3_stream *stream, { src->blksize = xd3_pow2_roundup(src->blksize); xd3_check_pow2 (src->blksize, &shiftby); + IF_DEBUG1 (DP(RINT "raising src_blksz to %"W"u\n", src->blksize)); } src->shiftby = shiftby; src->maskby = (1ULL << shiftby) - 1ULL; if (xd3_check_pow2 (src->max_winsize, NULL) != 0) + { src->max_winsize = xd3_xoff_roundup(src->max_winsize); + IF_DEBUG1 (DP(RINT "raising src_maxsize to %"W"u\n", src->blksize)); + } src->max_winsize = xd3_max (src->max_winsize, XD3_ALLOCSIZE); return 0; } @@ -1842,10 +1959,15 @@ xd3_set_source_and_size (xd3_stream *stream, if (ret == 0) { stream->src->eof_known = 1; + IF_DEBUG2 (DP(RINT "[set source] size known %"Q"u\n", + source_size)); xd3_blksize_div(source_size, stream->src, &stream->src->max_blkno, &stream->src->onlastblk); + + IF_DEBUG1 (DP(RINT "[set source] size known %"Q"u max_blkno %"Q"u\n", + source_size, stream->src->max_blkno)); } return ret; } @@ -1947,6 +2069,17 @@ xd3_set_appheader (xd3_stream *stream, stream->enc_appheadsz = size; } +#if XD3_DEBUG +static int +xd3_iopt_check (xd3_stream *stream) +{ + usize_t ul = xd3_rlist_length (& stream->iopt_used); + usize_t fl = xd3_rlist_length (& stream->iopt_free); + + return (ul + fl + (stream->iout ? 1 : 0)) == stream->iopt_size; +} +#endif + static xd3_rinst* xd3_iopt_free (xd3_stream *stream, xd3_rinst *i) { @@ -1973,6 +2106,9 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) { int ret; + /* Check for input overflow. */ + XD3_ASSERT (inst->pos + inst->size <= stream->avail_in); + switch (inst->type) { case XD3_CPY: @@ -2001,6 +2137,9 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) /* xtra field indicates the copy is from the source */ if (inst->xtra) { + XD3_ASSERT (inst->addr >= src->srcbase); + XD3_ASSERT (inst->addr + inst->size <= + src->srcbase + src->srclen); addr = inst->addr - src->srcbase; stream->n_scpy += 1; stream->l_scpy += inst->size; @@ -2026,8 +2165,18 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) /* the "here" position is always offset by taroff */ if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, & inst->type))) + { return ret; + } + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt copy:%d] pos %"Q"u-%"Q"u addr %"Q"u-%"Q"u size %"W"u\n", + cnt++, + stream->total_in + inst->pos, + stream->total_in + inst->pos + inst->size, + inst->addr, inst->addr + inst->size, inst->size); + }); break; } case XD3_RUN: @@ -2037,6 +2186,10 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) stream->n_run += 1; stream->l_run += inst->size; + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt run:%d] pos %"Q"u size %"W"u\n", cnt++, stream->total_in + inst->pos, inst->size); + }); break; } case XD3_ADD: @@ -2047,11 +2200,17 @@ xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) stream->n_add += 1; stream->l_add += inst->size; + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt add:%d] pos %"Q"u size %"W"u\n", cnt++, stream->total_in + inst->pos, inst->size); + }); + break; } } /* This is the only place stream->unencoded_offset is incremented. */ + XD3_ASSERT (stream->unencoded_offset == inst->pos); stream->unencoded_offset += inst->size; inst->code2 = 0; @@ -2158,6 +2317,8 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) usize_t flushed; int ret; + XD3_ASSERT (xd3_iopt_check (stream)); + /* Note: once tried to skip this step if it's possible to assert * there are no overlapping instructions. Doesn't work because * xd3_opt_erase leaves overlapping instructions. */ @@ -2175,6 +2336,9 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) r2end = r2->pos + r2->size; + /* The min_match adjustments prevent this. */ + XD3_ASSERT (r2end > (r1end + LEAST_MATCH_INCR)); + /* If r3 is available... */ if (! xd3_rlist_end (& stream->iopt_used, r3 = xd3_rlist_next (r2))) { @@ -2224,6 +2388,8 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) usize_t newsize; usize_t adjust1; + XD3_ASSERT (r1end > r2->pos && r2end > r1->pos); + /* Try to balance the length of both instructions, but avoid * making both longer than MAX_MATCH_SPLIT . */ average = gap / 2; @@ -2240,6 +2406,8 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) /* shorten r1 */ adjust1 = r1end - r2->pos; + XD3_ASSERT (r1->size > adjust1); + r1->size -= adjust1; /* don't shorten r2 */ @@ -2251,21 +2419,36 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) adjust1 = r1->size - newsize; if (r2->pos > r1end - adjust1) + { adjust1 -= r2->pos - (r1end - adjust1); + } + + XD3_ASSERT (r1->size > adjust1); r1->size -= adjust1; + /* shorten r2 */ + XD3_ASSERT (r1->pos + r1->size >= r2->pos); + adjust1 = r1->pos + r1->size - r2->pos; } + /* Fallthrough above if-else, shorten r2 */ + XD3_ASSERT (r2->size > adjust1); + r2->size -= adjust1; r2->pos += adjust1; r2->addr += adjust1; + XD3_ASSERT (r1->size >= MIN_MATCH); + XD3_ASSERT (r2->size >= MIN_MATCH); + r1 = r2; } } + XD3_ASSERT (xd3_iopt_check (stream)); + /* If forcing, pick instructions until the list is empty, otherwise * this empties 50% of the queue. */ for (flushed = 0; ! xd3_rlist_empty (& stream->iopt_used); ) @@ -2296,6 +2479,10 @@ xd3_iopt_flush_instructions (xd3_stream *stream, int force) } } + XD3_ASSERT (xd3_iopt_check (stream)); + + XD3_ASSERT (!force || xd3_rlist_length (& stream->iopt_used) == 0); + return 0; } @@ -2321,6 +2508,8 @@ xd3_iopt_get_slot (xd3_stream *stream, xd3_rinst** iptr) else { if ((ret = xd3_iopt_flush_instructions (stream, 0))) { return ret; } + + XD3_ASSERT (! xd3_rlist_empty (& stream->iopt_free)); } } @@ -2347,6 +2536,13 @@ xd3_iopt_erase (xd3_stream *stream, usize_t pos, usize_t size) { xd3_rinst *r = xd3_rlist_back (& stream->iopt_used); + /* Verify that greedy is working. The previous instruction + * should end before the new one begins. */ + XD3_ASSERT ((stream->flags & XD3_BEGREEDY) == 0 || (r->pos + r->size <= pos)); + /* Verify that min_match is working. The previous instruction + * should end before the new one ends. */ + XD3_ASSERT ((stream->flags & XD3_BEGREEDY) != 0 || (r->pos + r->size < pos + size)); + /* See if the last instruction starts before the new * instruction. If so, there is nothing to erase. */ if (r->pos < pos) @@ -2388,13 +2584,23 @@ xd3_emit_single (xd3_stream *stream, xd3_rinst *single, uint8_t code) int has_size = stream->code_table[code].size1 == 0; int ret; + IF_DEBUG2 (DP(RINT "[emit1] %"W"u %s (%"W"u) code %u\n", + single->pos, + xd3_rtype_to_string ((xd3_rtype) single->type, 0), + single->size, + code)); + if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) + { return ret; + } if (has_size) { if ((ret = xd3_emit_size (stream, & INST_TAIL (stream), single->size))) + { return ret; + } } return 0; @@ -2406,8 +2612,23 @@ xd3_emit_double (xd3_stream *stream, xd3_rinst *first, { int ret; + /* All double instructions use fixed sizes, so all we need to do is + * output the instruction code, no sizes. */ + XD3_ASSERT (stream->code_table[code].size1 != 0 && + stream->code_table[code].size2 != 0); + if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) + { return ret; + } + + IF_DEBUG2 (DP(RINT "[emit2]: %"W"u %s (%"W"u) %s (%"W"u) code %u\n", + first->pos, + xd3_rtype_to_string ((xd3_rtype) first->type, 0), + first->size, + xd3_rtype_to_string ((xd3_rtype) second->type, 0), + second->size, + code)); return 0; } @@ -2631,9 +2852,17 @@ xd3_encode_buffer_leftover (xd3_stream *stream) return ENOMEM; } + IF_DEBUG2 (DP(RINT "[leftover] flush?=%s\n", (stream->flags & XD3_FLUSH) ? "yes" : "no")); + /* Take leftover input first. */ if (stream->buf_leftover != NULL) { + XD3_ASSERT (stream->buf_avail == 0); + XD3_ASSERT (stream->buf_leftavail < stream->winsize); + + IF_DEBUG2 (DP(RINT "[leftover] previous %"W"u avail %"W"u\n", + stream->buf_leftavail, stream->avail_in)); + memcpy (stream->buf_in, stream->buf_leftover, stream->buf_leftavail); stream->buf_leftover = NULL; @@ -2657,10 +2886,12 @@ xd3_encode_buffer_leftover (xd3_stream *stream) else if ((stream->buf_avail < stream->winsize) && !(stream->flags & XD3_FLUSH)) { /* Buffer has space */ + IF_DEBUG2 (DP(RINT "[leftover] emptied %"W"u\n", take)); return XD3_INPUT; } /* Use the buffer: */ + IF_DEBUG2 (DP(RINT "[leftover] take %"W"u remaining %"W"u\n", take, stream->buf_leftavail)); stream->next_in = stream->buf_in; stream->avail_in = stream->buf_avail; stream->buf_avail = 0; @@ -2757,6 +2988,9 @@ xd3_encode_init (xd3_stream *stream, int full_init) if (xd3_alloc_iopt (stream, stream->iopt_size) != 0) { goto fail; } + XD3_ASSERT (xd3_rlist_length (& stream->iopt_free) == stream->iopt_size); + XD3_ASSERT (xd3_rlist_length (& stream->iopt_used) == 0); + /* address cache, code table */ stream->acache.s_near = stream->code_table_desc->near_modes; stream->acache.s_same = stream->code_table_desc->same_modes; @@ -2810,6 +3044,8 @@ xd3_encode_reset (xd3_stream *stream) for (i = 0; i < ENC_SECTS; i += 1) { + XD3_ASSERT (olist != NULL); + stream->enc_heads[i] = olist; stream->enc_tails[i] = olist; olist = olist->next_page; @@ -2880,9 +3116,15 @@ xd3_encode_input (xd3_stream *stream) stream->enc_state = ENC_SEARCH; + IF_DEBUG2 (DP(RINT "[WINSTART:%"Q"u] input bytes %"W"u offset %"Q"u\n", + stream->current_window, stream->avail_in, + stream->total_in)); return XD3_WINSTART; case ENC_SEARCH: + IF_DEBUG2 (DP(RINT "[SEARCH] match_state %d avail_in %"W"u %s\n", + stream->match_state, stream->avail_in, + stream->src ? "source" : "no source")); /* Reentrant matching. */ if (stream->src != NULL) @@ -2902,6 +3144,7 @@ xd3_encode_input (xd3_stream *stream) /* This call can't fail because the source window is * unrestricted. */ ret = xd3_source_match_setup (stream, stream->match_srcpos); + XD3_ASSERT (ret == 0); stream->match_state = MATCH_FORWARD; } else @@ -2909,6 +3152,7 @@ xd3_encode_input (xd3_stream *stream) stream->match_state = MATCH_SEARCHING; stream->match_fwd = 0; } + XD3_ASSERT (stream->match_fwd == 0); case MATCH_FORWARD: case MATCH_BACKWARD: @@ -3014,6 +3258,9 @@ xd3_encode_input (xd3_stream *stream) stream->total_in += stream->avail_in; stream->enc_state = ENC_POSTWIN; + IF_DEBUG2 (DP(RINT "[WINFINISH:%"Q"u] in=%"Q"u\n", + stream->current_window, + stream->total_in)); return XD3_WINFINISH; case ENC_POSTWIN: @@ -3184,6 +3431,10 @@ xd3_process_memory (int is_encode, } exit: + if (ret != 0) + { + IF_DEBUG2 (DP(RINT "process_memory: %d: %s\n", ret, stream.msg)); + } xd3_free_stream(&stream); return ret; } @@ -3356,6 +3607,9 @@ xd3_srcwin_setup (xd3_stream *stream) xd3_source *src = stream->src; xoff_t length, x; + /* Check the undecided state. */ + XD3_ASSERT (src->srclen == 0 && src->srcbase == 0); + /* Avoid repeating this call. */ stream->srcwin_decided = 1; @@ -3387,6 +3641,7 @@ xd3_srcwin_setup (xd3_stream *stream) { src->srcbase = stream->match_minaddr; src->srclen = (usize_t) length; + XD3_ASSERT (src->srclen); goto done; } @@ -3406,7 +3661,10 @@ xd3_srcwin_setup (xd3_stream *stream) * for the second block. */ src->srclen = xd3_min (src->srclen, xd3_source_eof(src) - src->srcbase); } + IF_DEBUG1 (DP(RINT "[srcwin_setup_constrained] base %"Q"u len %"W"u\n", + src->srcbase, src->srclen)); + XD3_ASSERT (src->srclen); done: /* Set the taroff. This convenience variable is used even when stream->src == NULL. */ @@ -3440,14 +3698,32 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) * won't suffice to avoid loops. See testing/regtest.cc's * TestNonBlockingProgress test! */ if (srcpos != 0 && srcpos == stream->match_last_srcpos) + { + IF_DEBUG2(DP(RINT "[match_setup] looping failure\n")); goto bad; + } /* Implement src->max_winsize, which prevents the encoder from seeking * back further than the LRU cache maintaining FIFO discipline, (to * avoid seeking). */ if (srcpos < stream->srcwin_cksum_pos && stream->srcwin_cksum_pos - srcpos > src->max_winsize) + { + IF_DEBUG2(DP(RINT "[match_setup] rejected due to src->max_winsize " + "distance eof=%"Q"u srcpos=%"Q"u max_winsz=%"Q"u\n", + xd3_source_eof (src), + srcpos, src->max_winsize)); goto bad; + } + + /* There are cases where the above test does not reject a match that + * will experience XD3_TOOFARBACK at the first xd3_getblk call + * because the input may have advanced up to one block beyond the + * actual EOF. */ + IF_DEBUG2(DP(RINT "[match_setup] %"Q"u srcpos %"Q"u, " + "src->max_winsize %"Q"u\n", + stream->total_in + stream->input_position, + srcpos, src->max_winsize)); /* Going backwards, the 1.5-pass algorithm allows some * already-matched input may be covered by a longer source match. @@ -3470,9 +3746,11 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) } /* Backward target match limit. */ + XD3_ASSERT (stream->input_position >= greedy_or_not); stream->match_maxback = stream->input_position - greedy_or_not; /* Forward target match limit. */ + XD3_ASSERT (stream->avail_in > stream->input_position); stream->match_maxfwd = stream->avail_in - stream->input_position; /* Now we take the source position into account. It depends whether @@ -3494,16 +3772,31 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) xoff_t srcavail = xd3_source_eof (src) - srcpos; if (srcavail < stream->match_maxfwd) + { stream->match_maxfwd = (usize_t) srcavail; + } } + IF_DEBUG2(DP(RINT + "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "unrestricted maxback %"W"u maxfwd %"W"u\n", + srcpos, + stream->total_in + stream->input_position, + stream->match_maxback, + stream->match_maxfwd)); goto good; } + /* Decided some source window. */ + XD3_ASSERT (src->srclen > 0); + /* Restricted case: fail if the srcpos lies outside the source window */ if ((srcpos < src->srcbase) || (srcpos > (src->srcbase + src->srclen))) + { + IF_DEBUG1(DP(RINT "[match_setup] restricted source window failure\n")); goto bad; + } else { usize_t srcavail; @@ -3516,8 +3809,17 @@ xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) srcavail = src->srcbase + src->srclen - srcpos; if (srcavail < stream->match_maxfwd) + { stream->match_maxfwd = srcavail; + } + IF_DEBUG2(DP(RINT + "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "restricted maxback %"W"u maxfwd %"W"u\n", + srcpos, + stream->total_in + stream->input_position, + stream->match_maxback, + stream->match_maxfwd)); goto good; } @@ -3592,6 +3894,11 @@ xd3_source_extend_match (xd3_stream *stream) usize_t tryrem; /* tryrem is the number of matchable bytes */ usize_t matched; + IF_DEBUG2(DP(RINT "[extend match] srcpos %"Q"u\n", + stream->match_srcpos)); + + XD3_ASSERT (src != NULL); + /* Does it make sense to compute backward match AFTER forward match? */ if (stream->match_state == MATCH_BACKWARD) { @@ -3615,9 +3922,17 @@ xd3_source_extend_match (xd3_stream *stream) { if (ret == XD3_TOOFARBACK) { + IF_DEBUG2(DP(RINT "[maxback] %"Q"u TOOFARBACK: %"W"u INP %"Q"u CKSUM %"Q"u\n", + tryblk, stream->match_back, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos)); + /* the starting position is too far back. */ if (stream->match_back == 0) + { + XD3_ASSERT(stream->match_fwd == 0); goto donefwd; + } /* search went too far back, continue forward. */ goto doneback; @@ -3629,6 +3944,9 @@ xd3_source_extend_match (xd3_stream *stream) tryrem = xd3_min (tryoff, stream->match_maxback - stream->match_back); + IF_DEBUG2(DP(RINT "[maxback] maxback %"W"u trysrc %"Q"u/%"W"u tgt %"W"u tryrem %"W"u\n", + stream->match_maxback, tryblk, tryoff, streamoff, tryrem)); + /* TODO: This code can be optimized similar to xd3_match_forward() */ for (; tryrem != 0; tryrem -= 1, stream->match_back += 1) { @@ -3646,6 +3964,8 @@ xd3_source_extend_match (xd3_stream *stream) stream->match_state = MATCH_FORWARD; } + XD3_ASSERT (stream->match_state == MATCH_FORWARD); + matchoff = stream->match_srcpos + stream->match_fwd; streamoff = stream->input_position + stream->match_fwd; xd3_blksize_div (matchoff, src, & tryblk, & tryoff); @@ -3662,7 +3982,13 @@ xd3_source_extend_match (xd3_stream *stream) if ((ret = xd3_getblk (stream, tryblk))) { if (ret == XD3_TOOFARBACK) + { + IF_DEBUG2(DP(RINT "[maxfwd] %"Q"u TOOFARBACK: %"W"u INP %"Q"u CKSUM %"Q"u\n", + tryblk, stream->match_fwd, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos)); goto donefwd; + } /* could be a XD3_GETSRCBLK failure. */ return ret; @@ -3671,11 +3997,14 @@ xd3_source_extend_match (xd3_stream *stream) tryrem = xd3_min(stream->match_maxfwd - stream->match_fwd, src->onblk - tryoff); - /* Generally, this means we have a power-of-two size source - * and we just found the end-of-file, in this case it's an - * empty block. */ if (tryrem == 0) + { + /* Generally, this means we have a power-of-two size source + * and we just found the end-of-file, in this case it's an + * empty block. */ + XD3_ASSERT (src->onblk < src->blksize); break; + } matched = xd3_forward_match(src->curblk + tryoff, stream->next_in + streamoff, @@ -3693,6 +4022,11 @@ xd3_source_extend_match (xd3_stream *stream) donefwd: stream->match_state = MATCH_SEARCHING; + IF_DEBUG2(DP(RINT "[extend match] input %"Q"u srcpos %"Q"u len %"W"u\n", + stream->input_position + stream->total_in, + stream->match_srcpos, + stream->match_fwd)); + /* If the match ends short of the last instruction end, we probably * don't want it. There is the possibility that a copy ends short * of the last copy but also goes further back, in which case we @@ -3742,6 +4076,18 @@ xd3_source_extend_match (xd3_stream *stream) stream->maxsrcaddr = match_end; } + IF_DEBUG2 ({ + static int x = 0; + DP(RINT "[source match:%d] length %"W"u (%s)\n", + x++, + match_length, + stream->total_in + target_position, + stream->total_in + target_position + match_length, + match_position, + match_position + match_length, + (stream->total_in + target_position == match_position) ? "same" : "diff"); + }); + if ((ret = xd3_found_match (stream, /* decoder position */ target_position, /* length */ match_length, @@ -3785,6 +4131,27 @@ xd3_scksum_insert (xd3_stream *stream, stream->small_table[inx] = pos + HASH_CKOFFSET; } +#if XD3_DEBUG +static int +xd3_check_smatch (const uint8_t *ref0, const uint8_t *inp0, + const uint8_t *inp_max, usize_t cmp_len) +{ + usize_t i; + + for (i = 0; i < cmp_len; i += 1) + { + XD3_ASSERT (ref0[i] == inp0[i]); + } + + if (inp0 + cmp_len < inp_max) + { + XD3_ASSERT (inp0[i] != ref0[i]); + } + + return 1; +} +#endif /* XD3_DEBUG */ + /* When the hash table indicates a possible small string match, it * calls this routine to find the best match. The first matching * position is taken from the small_table, HASH_CKOFFSET is subtracted @@ -3808,13 +4175,26 @@ xd3_smatch (xd3_stream *stream, const uint8_t *inp; const uint8_t *ref; + SMALL_HASH_DEBUG1 (stream, stream->next_in + stream->input_position); + + XD3_ASSERT (stream->min_match + stream->input_position <= stream->avail_in); + base -= HASH_CKOFFSET; again: + IF_DEBUG2 (DP(RINT "smatch at base=%"W"u inp=%"W"u cksum=%"W"u\n", base, + stream->input_position, scksum)); + + /* For small matches, we can always go to the end-of-input because + * the matching position must be less than the input position. */ + XD3_ASSERT (base < stream->input_position); + ref = stream->next_in + base; inp = stream->next_in + stream->input_position; + SMALL_HASH_DEBUG2 (stream, ref); + /* Expand potential match forward. */ while (inp < inp_max && *inp == *ref) { @@ -3824,6 +4204,11 @@ xd3_smatch (xd3_stream *stream, cmp_len = (usize_t)(inp - (stream->next_in + stream->input_position)); + /* Verify correctness */ + XD3_ASSERT (xd3_check_smatch (stream->next_in + base, + stream->next_in + stream->input_position, + inp_max, cmp_len)); + /* Update longest match */ if (cmp_len > match_length) { @@ -3859,6 +4244,7 @@ xd3_smatch (xd3_stream *stream, base = prev_pos; + XD3_ASSERT (stream->input_position > base); diff_pos = stream->input_position - base; /* Stop searching if we go beyond sprevsz, since those entries @@ -3894,6 +4280,41 @@ xd3_smatch (xd3_stream *stream, return match_length; } +#if XD3_DEBUG +static void +xd3_verify_small_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum) +{ + uint32_t state; + uint32_t cksum = xd3_scksum (&state, inp, stream->smatcher.small_look); + + XD3_ASSERT (cksum == x_cksum); +} + +static void +xd3_verify_large_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_cksum) +{ + usize_t cksum = xd3_large_cksum (&stream->large_hash, inp, stream->smatcher.large_look); + XD3_ASSERT (cksum == x_cksum); +} +static void +xd3_verify_run_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_run_l, + uint8_t *x_run_c) +{ + usize_t slook = stream->smatcher.small_look; + uint8_t run_c; + usize_t run_l = xd3_comprun (inp, slook, &run_c); + + XD3_ASSERT (run_l == 0 || run_c == *x_run_c); + XD3_ASSERT (x_run_l > slook || run_l == x_run_l); +} +#endif /* XD3_DEBUG */ + /* This function computes more source checksums to advance the window. * Called at every entrance to the string-match loop and each time * stream->input_position reaches the value returned as @@ -3911,6 +4332,7 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) if (stream->src->eof_known) { xoff_t source_size = xd3_source_eof (stream->src); + XD3_ASSERT(stream->srcwin_cksum_pos <= source_size); if (stream->srcwin_cksum_pos == source_size) { @@ -3977,16 +4399,31 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) { /* TOOFARBACK should never occur here, since we read forward. */ if (ret == XD3_TOOFARBACK) + { ret = XD3_INTERNAL; + } + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] async getblk return for %"Q"u: %s\n", + blkno, xd3_strerror (ret))); return ret; } + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] block %"Q"u T=%"Q"u S=%"Q"u L=%"Q"u EOF=%"Q"u %s\n", + blkno, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos, + target_cksum_pos, + xd3_source_eof (stream->src), + stream->src->eof_known ? "known" : "unknown")); + blkpos = xd3_bytes_on_srcblk (stream->src, blkno); if (blkpos < (ssize_t) stream->smatcher.large_look) { stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; + IF_DEBUG2 (DP(RINT "[srcwin_move_point] continue (end-of-block): %"Z"d\n", blkpos)); continue; } @@ -4016,6 +4453,8 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) (usize_t) (blkbaseoffset + (xoff_t)(blkpos + HASH_CKOFFSET)); + IF_DEBUG (stream->large_ckcnt += 1); + blkpos -= stream->smatcher.large_step; } while (blkpos >= oldpos); @@ -4023,6 +4462,14 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; } + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] exited loop T=%"Q"u " + "S=%"Q"u EOF=%"Q"u %s\n", + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos, + xd3_source_eof (stream->src), + stream->src->eof_known ? "known" : "unknown")); + if (stream->src->eof_known) { xoff_t source_size = xd3_source_eof (stream->src); @@ -4031,14 +4478,29 @@ xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) /* This invariant is needed for xd3_source_cksum_offset() */ stream->srcwin_cksum_pos = source_size; *next_move_point = USIZE_T_MAX; + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] finished with source input\n")); return 0; } } + /* How long until this function should be called again. */ + XD3_ASSERT(stream->srcwin_cksum_pos >= target_cksum_pos); + *next_move_point = stream->input_position + stream->src->blksize - ((stream->srcwin_cksum_pos - target_cksum_pos) & stream->src->maskby); + IF_DEBUG2 (DP(RINT + "[srcwin_move_point] finished T=%"Q"u " + "S=%"Q"u L=%"Q"u EOF=%"Q"u %s again in %"W"u\n", + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos, + target_cksum_pos, + xd3_source_eof (stream->src), + stream->src->eof_known ? "known" : "unknown", + *next_move_point - stream->input_position)); + return 0; } @@ -4097,6 +4559,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) usize_t match_offset = 0; usize_t next_move_point = 0; + IF_DEBUG2(DP(RINT "[string_match] initial entry %"W"u\n", stream->input_position)); + /* If there will be no compression due to settings or short input, * skip it entirely. */ if (! (DO_SMALL || DO_LARGE || DO_RUN) || @@ -4108,6 +4572,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) * needs to be reset. */ restartloop: + IF_DEBUG2(DP(RINT "[string_match] restartloop %"W"u\n", stream->input_position)); + /* If there is not enough input remaining for any kind of match, skip it. */ if (stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } @@ -4196,6 +4662,8 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) { usize_t max_len = stream->avail_in - stream->input_position; + IF_DEBUG (xd3_verify_run_state (stream, inp, run_l, &run_c)); + while (run_l < max_len && inp[run_l] == run_c) { run_l += 1; } /* Output a RUN instruction. */ @@ -4213,10 +4681,14 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) { if ((stream->input_position >= next_move_point) && (ret = xd3_srcwin_move_point (stream, & next_move_point))) + { return ret; + } linx = xd3_checksum_hash (& stream->large_hash, lcksum); + IF_DEBUG (xd3_verify_large_state (stream, inp, lcksum)); + if (stream->large_table[linx] != 0) { /* the match_setup will fail if the source window has @@ -4249,6 +4721,9 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) { sinx = xd3_checksum_hash (& stream->small_hash, scksum); + /* Verify incremental state in debugging mode. */ + IF_DEBUG (xd3_verify_small_state (stream, inp, scksum)); + /* Search for the longest match */ if (stream->small_table[sinx] != 0) { @@ -4268,13 +4743,28 @@ XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) /* Maybe output a COPY instruction */ if (match_length >= stream->min_match) { + IF_DEBUG2 ({ + static int x = 0; + DP(RINT "[target match:%d] " + "(-%"W"d) [ %"W"u bytes ]\n", + x++, + stream->input_position, + stream->input_position + match_length, + match_offset, + match_offset + match_length, + stream->input_position - match_offset, + match_length); + }); + if ((ret = xd3_found_match (stream, /* decoder position */ stream->input_position, /* length */ match_length, /* address */ (xoff_t) match_offset, /* is_source */ 0))) + { return ret; + } /* Copy instruction. */ HANDLELAZY (match_length); diff --git a/deps/xdelta3/xdelta3.h b/deps/xdelta3/xdelta3.h index 8ce2929551..34bf5c3aba 100644 --- a/deps/xdelta3/xdelta3.h +++ b/deps/xdelta3/xdelta3.h @@ -22,11 +22,21 @@ #ifndef _XDELTA3_H_ #define _XDELTA3_H_ +#define _POSIX_SOURCE 200112L +#define _ISOC99_SOURCE +#define _C99_SOURCE /* To include RetroArch's INLINE macro */ -#include +#include "retro_inline.h" +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include #include #include +#include #include #include #include @@ -106,9 +116,46 @@ #ifndef _WIN32 #define __STDC_FORMAT_MACROS #include -#endif /* _WIN32 defined */ - #include +#else /* WIN32 case */ +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#ifndef WINVER +#if XD3_USE_LARGEFILE64 +/* 64 bit file offsets: uses GetFileSizeEx and SetFilePointerEx. */ +#define WINVER 0x0500 +#define _WIN32_WINNT 0x0500 +#else /* xoff_t is 32bit */ +/* 32 bit file offsets: uses GetFileSize and SetFilePointer. */ +#define WINVER 0x0400 +#define _WIN32_WINNT 0x0400 +#endif /* if XD3_USE_LARGEFILE64 */ +#endif /* ifndef WINVER */ + +#include + +/* _MSV_VER is defined by Microsoft tools, not by Mingw32 */ +#ifdef _MSC_VER +typedef signed int ssize_t; +typedef int pid_t; +#if _MSC_VER < 1600 +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned long uint32_t; +typedef ULONGLONG uint64_t; +#else /* _MSC_VER >= 1600 */ +/* For MSVC10 and above */ +#include +#define inline __inline +#endif /* _MSC_VER < 1600 */ +#else /* _MSC_VER not defined */ +/* Mingw32 */ +#include +#endif /* _MSC_VER defined */ + +#endif /* _WIN32 defined */ #if SIZE_MAX == UINT64_MAX #define SIZEOF_SIZE_T 8 @@ -149,6 +196,12 @@ #define _FILE_OFFSET_BITS 64 #endif +_Static_assert(SIZEOF_SIZE_T == sizeof(size_t), "SIZEOF_SIZE_T not correctly set"); + +#ifdef SIZEOF_UNSIGNED_LONG_LONG +_Static_assert(SIZEOF_UNSIGNED_LONG_LONG == sizeof(unsigned long long), "SIZEOF_UNSIGNED_LONG_LONG not correctly set"); +#endif + /* Set a xoff_t typedef and the "Q" printf insert. */ #if defined(_WIN32) typedef uint64_t xoff_t; @@ -229,8 +282,10 @@ typedef uint32_t usize_t; #error Bad configure script #endif /* size_t printf flags */ -#define USE_UINT32 (SIZEOF_USIZE_T == 4 || SIZEOF_XOFF_T == 4 ) -#define USE_UINT64 (SIZEOF_USIZE_T == 8 || SIZEOF_XOFF_T == 8 ) +#define USE_UINT32 (SIZEOF_USIZE_T == 4 || \ + SIZEOF_XOFF_T == 4 || REGRESSION_TEST) +#define USE_UINT64 (SIZEOF_USIZE_T == 8 || \ + SIZEOF_XOFF_T == 8 || REGRESSION_TEST) #ifndef UNALIGNED_OK #ifdef HAVE_ALIGNED_ACCESS_REQUIRED @@ -248,6 +303,37 @@ typedef uint32_t usize_t; #define XD3_ENCODER 1 #endif +/* The code returned when main() fails, also defined in system + includes. */ +#ifndef EXIT_FAILURE +#define EXIT_FAILURE 1 +#endif + +/* REGRESSION TEST enables the "xdelta3 test" command, which runs a + series of self-tests. */ +#ifndef REGRESSION_TEST +#define REGRESSION_TEST 0 +#endif + +/* XD3_DEBUG=1 enables assertions and various statistics. Levels > 1 + * enable some additional output only useful during development and + * debugging. */ +#ifndef XD3_DEBUG +#define XD3_DEBUG 0 +#endif + +#ifndef PYTHON_MODULE +#define PYTHON_MODULE 0 +#endif + +#ifndef SWIG_MODULE +#define SWIG_MODULE 0 +#endif + +#ifndef NOT_MAIN +#define NOT_MAIN 0 +#endif + /* There are three string matching functions supplied: one fast, one * slow (default), and one soft-configurable. To disable any of * these, use the following definitions. */ @@ -270,6 +356,10 @@ typedef uint32_t usize_t; #define XD3_BUILD_DEFAULT 1 #endif +#if XD3_DEBUG +#include +#endif + typedef struct _xd3_stream xd3_stream; typedef struct _xd3_source xd3_source; typedef struct _xd3_hash_cfg xd3_hash_cfg; @@ -310,6 +400,52 @@ typedef int (xd3_getblk_func) (xd3_stream *stream, typedef const xd3_dinst* (xd3_code_table_func) (void); + +#ifdef _WIN32 +#define vsnprintf_func _vsnprintf +#define snprintf_func _snprintf +#else +#define vsnprintf_func vsnprintf +#define snprintf_func snprintf +#endif + +/* Type used for short snprintf calls. */ +typedef struct { + char buf[48]; +} shortbuf; + +#ifndef PRINTF_ATTRIBUTE +#ifdef __GNUC__ +#define PRINTF_ATTRIBUTE(x,y) __attribute__ ((__format__ (__printf__, x, y))) +#else +#define PRINTF_ATTRIBUTE(x,y) +#endif +#endif + +/* Underlying xprintf() */ +int xsnprintf_func (char *str, size_t n, const char *fmt, ...) + PRINTF_ATTRIBUTE(3,4); + +/* XPR(NT "", ...) (used by main) prefixes an "xdelta3: " to the output. */ +void xprintf(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2); +#define XPR xprintf +#define NT "xdelta3: " +#define NTR "" +/* DP(RINT ...) */ +#define DP xprintf +#define RINT "" + +#if XD3_DEBUG +#define XD3_ASSERT(x) \ + do { \ + if (! (x)) { \ + DP(RINT "%s:%d: XD3 assertion failed: %s\n", \ + __FILE__, __LINE__, #x); \ + abort (); } } while (0) +#else +#define XD3_ASSERT(x) (void)0 +#endif /* XD3_DEBUG */ + #define xd3_max(x,y) ((x) < (y) ? (y) : (x)) #define xd3_min(x,y) ((x) < (y) ? (x) : (y)) @@ -990,6 +1126,14 @@ struct _xd3_stream xoff_t l_run; usize_t i_slots_used; + +#if XD3_DEBUG + usize_t large_ckcnt; + + /* memory usage */ + usize_t alloc_cnt; + usize_t free_cnt; +#endif }; /************************************************************************** @@ -1244,6 +1388,14 @@ void xd3_avail_input (xd3_stream *stream, const uint8_t *idata, usize_t isize) { + /* Even if isize is zero, the code expects a non-NULL idata. Why? + * It uses this value to determine whether xd3_avail_input has ever + * been called. If xd3_encode_input is called before + * xd3_avail_input it will return XD3_INPUT right away without + * allocating a stream->winsize buffer. This is to avoid an + * unwanted allocation. */ + XD3_ASSERT (idata != NULL || isize == 0); + stream->next_in = idata; stream->avail_in = isize; } @@ -1274,6 +1426,9 @@ usize_t xd3_encoder_srclen (xd3_stream *stream) { static INLINE void xd3_set_flags (xd3_stream *stream, uint32_t flags) { + /* The bitwise difference should contain only XD3_FLUSH or + XD3_SKIP_WINDOW */ + XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0); stream->flags = flags; } @@ -1296,6 +1451,7 @@ void xd3_blksize_div (const xoff_t offset, usize_t *blkoff) { *blkno = offset >> source->shiftby; *blkoff = offset & source->maskby; + XD3_ASSERT (*blkoff < source->blksize); } static INLINE @@ -1315,6 +1471,8 @@ void xd3_blksize_add (xoff_t *blkno, *blkno += blkdiff; *blkoff &= source->maskby; } + + XD3_ASSERT (*blkoff < source->blksize); } #ifdef __cplusplus @@ -1327,6 +1485,22 @@ void xd3_blksize_add (xoff_t *blkno, #define XD3_CPY 3U /* XD3_CPY rtypes are represented as (XD3_CPY + * copy-mode value) */ +#if XD3_DEBUG +#define IF_DEBUG(x) x +#else +#define IF_DEBUG(x) +#endif +#if XD3_DEBUG > 1 +#define IF_DEBUG1(x) x +#else +#define IF_DEBUG1(x) +#endif +#if XD3_DEBUG > 2 +#define IF_DEBUG2(x) x +#else +#define IF_DEBUG2(x) +#endif + #define SIZEOF_ARRAY(x) (sizeof(x) / sizeof(x[0])) #endif /* _XDELTA3_H_ */