From a566c9ab668a53c48df55f3ba8cd82605029c168 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Wed, 8 Jan 2020 19:58:40 -0800 Subject: [PATCH 01/20] Debugger: Fix tracing skipping instructions (fixes #1614) --- CHANGES | 1 + src/debugger/cli-debugger.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index c7462b175..f70a6c113 100644 --- a/CHANGES +++ b/CHANGES @@ -99,6 +99,7 @@ Other fixes: - 3DS: Fix screen darkening (fixes mgba.io/i/1562) - Core: Fix uninitialized memory issues with graphics caches - Core: Return null for out of bounds cached tile VRAM querying + - Debugger: Fix tracing skipping instructions (fixes mgba.io/i/1614) - OpenGL: Only invalidate texture if dimensions change (fixes mgba.io/i/1612) - Qt: Fix fast forward mute being reset (fixes mgba.io/i/1574) - Qt: Fix scrollbar arrows in memory view (fixes mgba.io/i/1558) diff --git a/src/debugger/cli-debugger.c b/src/debugger/cli-debugger.c index 35b5dd3a7..08054935a 100644 --- a/src/debugger/cli-debugger.c +++ b/src/debugger/cli-debugger.c @@ -709,7 +709,6 @@ static void _trace(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { static bool _doTrace(struct CLIDebugger* debugger) { char trace[1024]; trace[sizeof(trace) - 1] = '\0'; - debugger->d.core->step(debugger->d.core); size_t traceSize = sizeof(trace) - 2; debugger->d.platform->trace(debugger->d.platform, trace, &traceSize); if (traceSize + 1 <= sizeof(trace)) { From bf595be5c3fda6c84a9b7468b3bcce812596b355 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 9 Jan 2020 18:28:16 -0800 Subject: [PATCH 02/20] Tools: Clean up unused argument in mgba-perf --- CHANGES | 1 + src/platform/test/perf-main.c | 27 +++++++++++++++++---------- tools/perf.py | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index f70a6c113..ed106438f 100644 --- a/CHANGES +++ b/CHANGES @@ -119,6 +119,7 @@ Misc: - SDL: Use controller GUID instead of name - SM83: Rename LR35902 to SM83 - Tools: Allow using threaded renderer in perf.py + - Tools: Clean up unused argument in mgba-perf 0.8 beta 1: (2019-10-20) - Initial beta for 0.8 diff --git a/src/platform/test/perf-main.c b/src/platform/test/perf-main.c index feeab1a6d..97066fb36 100644 --- a/src/platform/test/perf-main.c +++ b/src/platform/test/perf-main.c @@ -63,12 +63,13 @@ static void _mPerfShutdown(int signal); static bool _parsePerfOpts(struct mSubParser* parser, int option, const char* arg); static void _log(struct mLogger*, int, enum mLogLevel, const char*, va_list); static bool _mPerfRunCore(const char* fname, const struct mArguments*, const struct PerfOpts*); -static bool _mPerfRunServer(const char* listen, const struct mArguments*, const struct PerfOpts*); +static bool _mPerfRunServer(const struct mArguments*, const struct PerfOpts*); static bool _dispatchExiting = false; static struct VFile* _savestate = 0; static void* _outputBuffer = NULL; static Socket _socket = INVALID_SOCKET; +static Socket _server = INVALID_SOCKET; int main(int argc, char** argv) { #ifdef _3DS @@ -100,7 +101,7 @@ int main(int argc, char** argv) { struct mArguments args = {}; bool parsed = parseArguments(&args, argc, argv, &subparser); - if (!args.fname) { + if (!args.fname && !perfOpts.server) { parsed = false; } if (!parsed || args.showHelp) { @@ -127,7 +128,7 @@ int main(int argc, char** argv) { #endif } if (perfOpts.server) { - didFail = !_mPerfRunServer(args.fname, &args, &perfOpts); + didFail = !_mPerfRunServer(&args, &perfOpts); } else { didFail = !_mPerfRunCore(args.fname, &args, &perfOpts); } @@ -266,19 +267,24 @@ static void _mPerfRunloop(struct mCore* core, int* frames, bool quiet) { } } -static bool _mPerfRunServer(const char* listen, const struct mArguments* args, const struct PerfOpts* perfOpts) { +static bool _mPerfRunServer(const struct mArguments* args, const struct PerfOpts* perfOpts) { SocketSubsystemInit(); - Socket server = SocketOpenTCP(7216, NULL); - if (SOCKET_FAILED(server)) { + _server = SocketOpenTCP(7216, NULL); + if (SOCKET_FAILED(_server)) { SocketSubsystemDeinit(); return false; } - if (SOCKET_FAILED(SocketListen(server, 0))) { - SocketClose(server); + if (SOCKET_FAILED(SocketListen(_server, 0))) { + SocketClose(_server); + SocketSubsystemDeinit(); + return false; + } + _socket = SocketAccept(_server, NULL); + if (SOCKET_FAILED(_socket)) { + SocketClose(_server); SocketSubsystemDeinit(); return false; } - _socket = SocketAccept(server, NULL); if (perfOpts->csv) { const char* header = "game_code,frames,duration,renderer\n"; SocketSend(_socket, header, strlen(header)); @@ -301,7 +307,7 @@ static bool _mPerfRunServer(const char* listen, const struct mArguments* args, c memset(path, 0, sizeof(path)); } SocketClose(_socket); - SocketClose(server); + SocketClose(_server); SocketSubsystemDeinit(); return true; } @@ -310,6 +316,7 @@ static void _mPerfShutdown(int signal) { UNUSED(signal); _dispatchExiting = true; SocketClose(_socket); + SocketClose(_server); } static bool _parsePerfOpts(struct mSubParser* parser, int option, const char* arg) { diff --git a/tools/perf.py b/tools/perf.py index 77b499042..a681385f3 100755 --- a/tools/perf.py +++ b/tools/perf.py @@ -90,7 +90,7 @@ class PerfServer(object): server_command = list(self.command) else: server_command = [os.path.join(os.getcwd(), PerfTest.EXECUTABLE)] - server_command.extend(['--', '-PD', '0']) + server_command.extend(['--', '-PD']) if hasattr(test, "frames"): server_command.extend(['-F', str(test.frames)]) if not test.renderer: From f96b08c52f6effe8cb882a2f4cf7b03c1dfec38e Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 9 Jan 2020 18:31:48 -0800 Subject: [PATCH 03/20] Debugger: Minor interface cleanup --- CHANGES | 1 + include/mgba/debugger/debugger.h | 5 ++--- src/debugger/cli-debugger.c | 6 +++--- src/debugger/debugger.c | 5 +++-- src/debugger/gdb-stub.c | 2 +- src/gb/core.c | 1 + src/gb/debugger/cli.c | 2 +- src/gba/core.c | 4 +--- src/gba/debugger/cli.c | 2 +- src/platform/python/mgba/debugger.py | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGES b/CHANGES index ed106438f..08ea80017 100644 --- a/CHANGES +++ b/CHANGES @@ -116,6 +116,7 @@ Misc: - GBA I/O: Stop logging several harmless invalid register reads - Debugger: Separate aliases from main commands - Debugger: Print break-/watchpoint ID when breaking in CLI + - Debugger: Minor interface cleanup - SDL: Use controller GUID instead of name - SM83: Rename LR35902 to SM83 - Tools: Allow using threaded renderer in perf.py diff --git a/include/mgba/debugger/debugger.h b/include/mgba/debugger/debugger.h index a3f548bc9..2a4bc2dce 100644 --- a/include/mgba/debugger/debugger.h +++ b/include/mgba/debugger/debugger.h @@ -20,17 +20,16 @@ extern const uint32_t DEBUGGER_ID; enum mDebuggerType { DEBUGGER_NONE = 0, + DEBUGGER_CUSTOM, DEBUGGER_CLI, -#ifdef USE_GDB_STUB DEBUGGER_GDB, -#endif DEBUGGER_MAX }; enum mDebuggerState { DEBUGGER_PAUSED, DEBUGGER_RUNNING, - DEBUGGER_CUSTOM, + DEBUGGER_CALLBACK, DEBUGGER_SHUTDOWN }; diff --git a/src/debugger/cli-debugger.c b/src/debugger/cli-debugger.c index 08054935a..474909270 100644 --- a/src/debugger/cli-debugger.c +++ b/src/debugger/cli-debugger.c @@ -155,7 +155,7 @@ static void _breakInto(struct CLIDebugger* debugger, struct CLIDebugVector* dv) static void _continue(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = debugger->traceRemaining != 0 ? DEBUGGER_CUSTOM : DEBUGGER_RUNNING; + debugger->d.state = debugger->traceRemaining != 0 ? DEBUGGER_CALLBACK : DEBUGGER_RUNNING; } static void _next(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { @@ -700,7 +700,7 @@ static void _trace(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { debugger->traceVf = VFileOpen(dv->next->charValue, O_CREAT | O_WRONLY | O_APPEND); } if (_doTrace(debugger)) { - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; } else { debugger->system->printStatus(debugger->system); } @@ -1043,7 +1043,7 @@ static void _cliDebuggerCustom(struct mDebugger* debugger) { if (cliDebugger->system) { retain = cliDebugger->system->custom(cliDebugger->system) && retain; } - if (!retain && debugger->state == DEBUGGER_CUSTOM) { + if (!retain && debugger->state == DEBUGGER_CALLBACK) { debugger->state = next; } } diff --git a/src/debugger/debugger.c b/src/debugger/debugger.c index 1ba186f52..675e04698 100644 --- a/src/debugger/debugger.c +++ b/src/debugger/debugger.c @@ -50,13 +50,14 @@ struct mDebugger* mDebuggerCreate(enum mDebuggerType type, struct mCore* core) { struct CLIDebuggerSystem* sys = core->cliDebuggerSystem(core); CLIDebuggerAttachSystem(&debugger->cli, sys); break; -#ifdef USE_GDB_STUB case DEBUGGER_GDB: +#ifdef USE_GDB_STUB GDBStubCreate(&debugger->gdb); GDBStubListen(&debugger->gdb, 2345, 0); break; #endif case DEBUGGER_NONE: + case DEBUGGER_CUSTOM: case DEBUGGER_MAX: free(debugger); return 0; @@ -89,7 +90,7 @@ void mDebuggerRun(struct mDebugger* debugger) { debugger->platform->checkBreakpoints(debugger->platform); } break; - case DEBUGGER_CUSTOM: + case DEBUGGER_CALLBACK: debugger->core->step(debugger->core); debugger->platform->checkBreakpoints(debugger->platform); debugger->custom(debugger); diff --git a/src/debugger/gdb-stub.c b/src/debugger/gdb-stub.c index 8a072f1e4..10c802108 100644 --- a/src/debugger/gdb-stub.c +++ b/src/debugger/gdb-stub.c @@ -205,7 +205,7 @@ static void _writeHostInfo(struct GDBStub* stub) { } static void _continue(struct GDBStub* stub, const char* message) { - stub->d.state = DEBUGGER_CUSTOM; + stub->d.state = DEBUGGER_CALLBACK; stub->untilPoll = GDB_STUB_INTERVAL; // TODO: parse message UNUSED(message); diff --git a/src/gb/core.c b/src/gb/core.c index 373dcf050..3647a043b 100644 --- a/src/gb/core.c +++ b/src/gb/core.c @@ -734,6 +734,7 @@ void* _GBGetMemoryBlock(struct mCore* core, size_t id, size_t* sizeOut) { static bool _GBCoreSupportsDebuggerType(struct mCore* core, enum mDebuggerType type) { UNUSED(core); switch (type) { + case DEBUGGER_CUSTOM: case DEBUGGER_CLI: return true; default: diff --git a/src/gb/debugger/cli.c b/src/gb/debugger/cli.c index ee0212ef0..9ad6cdd7a 100644 --- a/src/gb/debugger/cli.c +++ b/src/gb/debugger/cli.c @@ -66,7 +66,7 @@ static bool _GBCLIDebuggerCustom(struct CLIDebuggerSystem* debugger) { static void _frame(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; struct GBCLIDebugger* gbDebugger = (struct GBCLIDebugger*) debugger->system; gbDebugger->frameAdvance = true; diff --git a/src/gba/core.c b/src/gba/core.c index c6cadd233..81c9a29d7 100644 --- a/src/gba/core.c +++ b/src/gba/core.c @@ -819,12 +819,10 @@ void* _GBAGetMemoryBlock(struct mCore* core, size_t id, size_t* sizeOut) { static bool _GBACoreSupportsDebuggerType(struct mCore* core, enum mDebuggerType type) { UNUSED(core); switch (type) { + case DEBUGGER_CUSTOM: case DEBUGGER_CLI: - return true; -#ifdef USE_GDB_STUB case DEBUGGER_GDB: return true; -#endif default: return false; } diff --git a/src/gba/debugger/cli.c b/src/gba/debugger/cli.c index 5e0ba6ec2..643a12b5c 100644 --- a/src/gba/debugger/cli.c +++ b/src/gba/debugger/cli.c @@ -65,7 +65,7 @@ static bool _GBACLIDebuggerCustom(struct CLIDebuggerSystem* debugger) { static void _frame(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; struct GBACLIDebugger* gbaDebugger = (struct GBACLIDebugger*) debugger->system; gbaDebugger->frameAdvance = true; diff --git a/src/platform/python/mgba/debugger.py b/src/platform/python/mgba/debugger.py index ed3d6aaa5..dbc6b2b8a 100644 --- a/src/platform/python/mgba/debugger.py +++ b/src/platform/python/mgba/debugger.py @@ -56,7 +56,7 @@ class NativeDebugger(IRunner): @property def paused(self): - return self._native.state in (lib.DEBUGGER_PAUSED, lib.DEBUGGER_CUSTOM) + return self._native.state in (lib.DEBUGGER_PAUSED, lib.DEBUGGER_CALLBACK) def use_core(self): return DebuggerCoreOwner(self) From 28383d4d319dd5c0db0682c5a8b544bb645dbf61 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 9 Jan 2020 18:33:00 -0800 Subject: [PATCH 04/20] Qt: Use QSet::values instead of ::toList --- src/platform/qt/CheatsModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/qt/CheatsModel.cpp b/src/platform/qt/CheatsModel.cpp index a2c535f0c..77e608ec8 100644 --- a/src/platform/qt/CheatsModel.cpp +++ b/src/platform/qt/CheatsModel.cpp @@ -184,7 +184,7 @@ QString CheatsModel::toString(const QModelIndexList& indices) const { std::sort(order.begin(), order.end()); for (int i : order) { mCheatSet* set = setOrder[i]; - QList indexOrdex = setIndices[set].toList(); + QList indexOrdex = setIndices[set].values(); std::sort(indexOrdex.begin(), indexOrdex.end()); for (size_t j : indexOrdex) { strings.append(*StringListGetPointer(&set->lines, j)); From ec4c91dbda0eb871948335b4b005f2b4ac280140 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 10 Jan 2020 21:20:20 -0800 Subject: [PATCH 05/20] GBA Video: Copy higher priority target 1 flag --- CHANGES | 1 + .../obj-semitrans-overwrite/baseline_0000.png | Bin 0 -> 4683 bytes .../obj-semitrans-overwrite/baseline_0001.png | Bin 0 -> 4683 bytes .../gba/blend/obj-semitrans-overwrite/test.mvl | Bin 0 -> 18169 bytes src/gba/renderers/software-obj.c | 8 ++++---- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 cinema/gba/blend/obj-semitrans-overwrite/baseline_0000.png create mode 100644 cinema/gba/blend/obj-semitrans-overwrite/baseline_0001.png create mode 100644 cinema/gba/blend/obj-semitrans-overwrite/test.mvl diff --git a/CHANGES b/CHANGES index 08ea80017..08af9bcff 100644 --- a/CHANGES +++ b/CHANGES @@ -95,6 +95,7 @@ Emulation fixes: - GBA Video: Fix OpenGL renderer 512x512 backgrounds (fixes mgba.io/i/1572) - GBA Video: Fix BLDY for semitransparent sprite on non-target-2 backgrounds - GBA Video: Fix effects blending improperly in some non-last windows + - GBA Video: Copy higher priority target 1 flag Other fixes: - 3DS: Fix screen darkening (fixes mgba.io/i/1562) - Core: Fix uninitialized memory issues with graphics caches diff --git a/cinema/gba/blend/obj-semitrans-overwrite/baseline_0000.png b/cinema/gba/blend/obj-semitrans-overwrite/baseline_0000.png new file mode 100644 index 0000000000000000000000000000000000000000..f57a5beaf534f5cb5e6d760c5914ad3c96ffeb2f GIT binary patch literal 4683 zcmb7|c{CK>`^N{BN+`vUv6hs5%~F=KFOei=X_OGM#K>-FqZHY9L$VIWAhM27lVuo2 zSqE8WFc`xa%$V`>`F?+Ye9!s*{`viJ?|t8U?s@L>-1EH7dG398+scId4|9rW&$?D$Ndfnti?d^t_P0 zdXmOjSzD`slJH~->Ow+p6^HdqNBFhE^;LX zYUPiobfyFvJ`=(AH-G*$Q>ERC2%|TlQ>+lkubH}kH7)4s#h}gTn}q@u<@6?xPnB&l z+WDG90~v)%!@3bpgc33$m^H<3KN0SQgSq^)JqQ zsE?a4|2^V*G$Pc`1F}T=ysHUMMh7a1!A2$cRUzMujS7L zmH(;g11$!AbZ@{lj!*!{??LTWU`%($_@+%30equo&DG*fD$DFuQkQnc<_}K>+zHEZ z|BVEGN%(y9nnYgit6HeTLEqJ!A~in`hk^02&Inkk(!?o~$AVSsnjfxCzIufG(w-@w zC+Xo}b z-MQK4*6s8_3OBO3>)Pf_yRcYYOoA&JG7z28H44@fWKQKCo0o3>%d}3~QuX0=sLmn8 zmFQ;5k5;nd7qFwE0;abi{OwCsJ}44gx9vz?Q}V5d^8uUH+?;L)_3g*i-q;QF0-iwL ze|e1Rr0wuO0{_J}>2oMk(gY<^ z+{beQt0e5L_oB?&Gv#_)&a=C)$wfz>ECaYquvYs2?r#mS#{DW3E7mTz2e92rsW&^* zctM;?n=sQ&qa0$grsld#ALJFnRKCDp#H&-C=(QX zjrL5pB-BBtm|qv(>?CcqF`!rU?5d!^h~|i8b|ClVPgaW&jeSwjBX zvhh@iX~qN>D3&;eDGgA>vkU5)UrZ@YRCZ#g<$Aho?snO=$#Y!tx;5wRLkeAD+SeXf(40$S`CR~P$84e)rS`W= z*fb-(bwCSL+{$?F4m%PO?GZRP`UYk;WU z)SQ_;12}!L@QDA+L;)9IKu^h0+u?q_iHvU1lN?t{s!_P%*{?%Cynfza>oy+--CAJr zrP>FMIC220MS$iXg`}c_OK#Yzhh5GrwJuZCpt%o;_Vh_Y+^ksAUHFv*;M&l~wx0f$ zy1kIAKwkNYB34nzT~yT7X8jN87Vj@DH*81j3cD5$w@PW*A!1*Zj^@VFXnJv%)N@O} zE**s&yY!)nE&9bphk6sfxYN8PAAB2uKH3db5t490rC7y=(N|bXAVzijDDIO-jN=-@ zq)llef;WmS1SzT_%YBkQJ$fte0;>hZJhPuTBbL2K>4{X;{e{9y@qIBd(U_(+*h9+vB?`DQcEcLK-*)0)q+CZTYKL@3 z+Kzte%NQXn zrp|Xj%KF~Kz6f3kdx}0Vh=gbmErweW%y{T+9W@3gJ=I-%t*;(Pe93%A|3S3W<6Gh1 zRz!4Gc5Y0Mi%)ONEujBe1Qb`jemf0aJn`JFtY6!yhk6Qde@FVY+~ooSncFB-F-_I6 z`Tir~@-jb(wa@|_QxyI+{J2Pw`9mSb1SOe-eqUlbzu`o8frgCd4j1wjRe30t3ta@l_^iPHbk4>xJ z71HYv?!4vLr&O6!je(WgZ!53hdrI93Y!}T-@nZA^P!e-zul#6z!1n9m)(qLc)n2Ad zg9snL>LRVKsFL~=9fNdsEq=*USB&6SwfB1Yd)y&kq__0o+l`0nLGs-(zO{F{G0$PS za+yW=?*_P3I8XP9>8a@Udzn;y*r&e$Q~4*w`45xgs}ryAP>Y`m_aDvS&Pw$(SC!AO zsX6*>d^p(2|3uM84jm%$!b+W{8pDL9>ix7izjV_z&Am(-n(871-fk!+txUYq<%sL4 zr~eQbKOm>2X)8YA!LS(nM{b&&eV_3qNND)fB0BwU+zl&tNfT{m95gjeTl%&1ftWGn zsQ-mFDel0hG&o)6b-F$4xTs#+P1Xiqs%77oI|ZJ+i&OD(S*50OA0f_P!;qhDQz?mE zX|p1M7hqdR=+&r`;L6TVC3j}Bu?8xs0nASfM~^1YdoW4IVNx|?LtZ&wOv68SqzJ9Y z(_fggEL>8qAbbk<<#OLzM`IF5nY&q53~yCOuS;4CY7JXjadrLetmo^}06Bp#O9$Vo zNOd}m!~eZpkiG!7RWSm~+skZWHs<4z&Z=oGZO48jjU8I;`} zZ_<*lg@7J_mI$ODjs2SOkLqn)Wr&0z=v?=5>b0R~!A9y5M!q{O% z6#pXUnEH%r#NS+rWCZ+@GRX~CI47+f>wa*0hS_CDD)i(OLUtYdEz6-_;S$y&W+cjDFARdklefU*x0#v0*IcxlYhQv&+6x_-)qZR#ci)ydS(e|Tfm z{Pp2A-A`pk{)5!P~*~ z-*~L^T0T==#OvgR)`TE{>%@_!qHDi$`t0zR&IStlJVa|3a2#K|HO3C8E4mwPc>G`l z)uaR1=}SRg=Ybd)dOpM`tu+hIM`@jymyGD8Fh^rF{C`*yB&bz2QhdvTfVHCv;qegt zth5VQ?IQqfS+X@U{GlK@`~c5YvNo*&gbRI8I27e7m>A!ePqH;gEx%WVI?jgawVKJ? z&x0_O*E3SotI|B1?Oq&fiOn2Ag?|rrnOJvIk?{?eY_YcCJtKbgc9ia;k}Ly_@34mO zRQJt=1d4ff3T;G?;ybegK09p**7+g=D&M0*6#(N8)vOX@b*%vDF@gY}U?{48JcDkV z$dPHF(UFlgSxH{tmA=?x(FkJ=7DHIy5cd%KYC$)$?zvW!`3v?Jjw2Ruy=f&L**RpC zz&g${=4x{Row13K9Ys@ZdVH+24$C3SVQ^w(M5Y0(O_4)hwZ4(5@-6(?js?w{HMm&j zql2Ti7Dc{LVoX549hU1r;*6rS4{47WaJLUV8a5A)>?|)qnT{q&Dqou}J zRgHCrMG_951VbJ?n5d$N5>Tb_yMJ3a6o)%LZe|(O&0L24mkg*mU(7ZA)5UiIIu%U! zMIS2&HBI-jW0XsVX^rmGD)!2p9>l_=8eLmNCb?D(!T<%Kg8gW~bOdO4^3Bqmrdsil zXu@9%P-CHAS`7ae&ddKbmJdBr@8Gsy8{3#!d8zF;A>kGwU8Y)Dw!R zU=_2hB$P4}Be~pazd5c{g8UMXKabskFkW@DQ9G(b$4r<0G zHjx=2TMR7-{rw;du4NZWd$R{b7sf|~W;-Rzro2x-IVqz(5Kf8T7jN6cUKjrE=&Sd+ zF@C2>RGSjqUu%~k*%BrzIJhnv5C+FQho?c8LCtXt_p+DW9$sJG??Qw`I)C(01ChG6 z38@tUi;}v|BT-}v@)GQdiO}ns_Fr}rmJl36uS*RPh^hy&FsvN7E04?9fGDwLrf^w$_lGf*_=6!Ru+t}hCP{rz4 z1(N+D_6t7un`wWcUXMp_(j6(DHjbOb2F0!6{zBe!oM9GuX~ZV7BQHpjfEAPEp?bX2 zqMCJs=(a0gzJut7hv!7H9t91*%QO)CL1Y&0>_)Z1@+yLlWVfR-C-v}`AL#Z%5qRBE z1M&|1j~O{sVAKFwG6`^N{BN+`vUv6hs5%~F=KFOei=X_OGM#K>-FqZHY9L$VIWAhM27lVuo2 zSqE8WFc`xa%$V`>`F?+Ye9!s*{`viJ?|t8U?s@L>-1EH7dG398+scId4|9rW&$?D$Ndfnti?d^t_P0 zdXmOjSzD`slJH~->Ow+p6^HdqNBFhE^;LX zYUPiobfyFvJ`=(AH-G*$Q>ERC2%|TlQ>+lkubH}kH7)4s#h}gTn}q@u<@6?xPnB&l z+WDG90~v)%!@3bpgc33$m^H<3KN0SQgSq^)JqQ zsE?a4|2^V*G$Pc`1F}T=ysHUMMh7a1!A2$cRUzMujS7L zmH(;g11$!AbZ@{lj!*!{??LTWU`%($_@+%30equo&DG*fD$DFuQkQnc<_}K>+zHEZ z|BVEGN%(y9nnYgit6HeTLEqJ!A~in`hk^02&Inkk(!?o~$AVSsnjfxCzIufG(w-@w zC+Xo}b z-MQK4*6s8_3OBO3>)Pf_yRcYYOoA&JG7z28H44@fWKQKCo0o3>%d}3~QuX0=sLmn8 zmFQ;5k5;nd7qFwE0;abi{OwCsJ}44gx9vz?Q}V5d^8uUH+?;L)_3g*i-q;QF0-iwL ze|e1Rr0wuO0{_J}>2oMk(gY<^ z+{beQt0e5L_oB?&Gv#_)&a=C)$wfz>ECaYquvYs2?r#mS#{DW3E7mTz2e92rsW&^* zctM;?n=sQ&qa0$grsld#ALJFnRKCDp#H&-C=(QX zjrL5pB-BBtm|qv(>?CcqF`!rU?5d!^h~|i8b|ClVPgaW&jeSwjBX zvhh@iX~qN>D3&;eDGgA>vkU5)UrZ@YRCZ#g<$Aho?snO=$#Y!tx;5wRLkeAD+SeXf(40$S`CR~P$84e)rS`W= z*fb-(bwCSL+{$?F4m%PO?GZRP`UYk;WU z)SQ_;12}!L@QDA+L;)9IKu^h0+u?q_iHvU1lN?t{s!_P%*{?%Cynfza>oy+--CAJr zrP>FMIC220MS$iXg`}c_OK#Yzhh5GrwJuZCpt%o;_Vh_Y+^ksAUHFv*;M&l~wx0f$ zy1kIAKwkNYB34nzT~yT7X8jN87Vj@DH*81j3cD5$w@PW*A!1*Zj^@VFXnJv%)N@O} zE**s&yY!)nE&9bphk6sfxYN8PAAB2uKH3db5t490rC7y=(N|bXAVzijDDIO-jN=-@ zq)llef;WmS1SzT_%YBkQJ$fte0;>hZJhPuTBbL2K>4{X;{e{9y@qIBd(U_(+*h9+vB?`DQcEcLK-*)0)q+CZTYKL@3 z+Kzte%NQXn zrp|Xj%KF~Kz6f3kdx}0Vh=gbmErweW%y{T+9W@3gJ=I-%t*;(Pe93%A|3S3W<6Gh1 zRz!4Gc5Y0Mi%)ONEujBe1Qb`jemf0aJn`JFtY6!yhk6Qde@FVY+~ooSncFB-F-_I6 z`Tir~@-jb(wa@|_QxyI+{J2Pw`9mSb1SOe-eqUlbzu`o8frgCd4j1wjRe30t3ta@l_^iPHbk4>xJ z71HYv?!4vLr&O6!je(WgZ!53hdrI93Y!}T-@nZA^P!e-zul#6z!1n9m)(qLc)n2Ad zg9snL>LRVKsFL~=9fNdsEq=*USB&6SwfB1Yd)y&kq__0o+l`0nLGs-(zO{F{G0$PS za+yW=?*_P3I8XP9>8a@Udzn;y*r&e$Q~4*w`45xgs}ryAP>Y`m_aDvS&Pw$(SC!AO zsX6*>d^p(2|3uM84jm%$!b+W{8pDL9>ix7izjV_z&Am(-n(871-fk!+txUYq<%sL4 zr~eQbKOm>2X)8YA!LS(nM{b&&eV_3qNND)fB0BwU+zl&tNfT{m95gjeTl%&1ftWGn zsQ-mFDel0hG&o)6b-F$4xTs#+P1Xiqs%77oI|ZJ+i&OD(S*50OA0f_P!;qhDQz?mE zX|p1M7hqdR=+&r`;L6TVC3j}Bu?8xs0nASfM~^1YdoW4IVNx|?LtZ&wOv68SqzJ9Y z(_fggEL>8qAbbk<<#OLzM`IF5nY&q53~yCOuS;4CY7JXjadrLetmo^}06Bp#O9$Vo zNOd}m!~eZpkiG!7RWSm~+skZWHs<4z&Z=oGZO48jjU8I;`} zZ_<*lg@7J_mI$ODjs2SOkLqn)Wr&0z=v?=5>b0R~!A9y5M!q{O% z6#pXUnEH%r#NS+rWCZ+@GRX~CI47+f>wa*0hS_CDD)i(OLUtYdEz6-_;S$y&W+cjDFARdklefU*x0#v0*IcxlYhQv&+6x_-)qZR#ci)ydS(e|Tfm z{Pp2A-A`pk{)5!P~*~ z-*~L^T0T==#OvgR)`TE{>%@_!qHDi$`t0zR&IStlJVa|3a2#K|HO3C8E4mwPc>G`l z)uaR1=}SRg=Ybd)dOpM`tu+hIM`@jymyGD8Fh^rF{C`*yB&bz2QhdvTfVHCv;qegt zth5VQ?IQqfS+X@U{GlK@`~c5YvNo*&gbRI8I27e7m>A!ePqH;gEx%WVI?jgawVKJ? z&x0_O*E3SotI|B1?Oq&fiOn2Ag?|rrnOJvIk?{?eY_YcCJtKbgc9ia;k}Ly_@34mO zRQJt=1d4ff3T;G?;ybegK09p**7+g=D&M0*6#(N8)vOX@b*%vDF@gY}U?{48JcDkV z$dPHF(UFlgSxH{tmA=?x(FkJ=7DHIy5cd%KYC$)$?zvW!`3v?Jjw2Ruy=f&L**RpC zz&g${=4x{Row13K9Ys@ZdVH+24$C3SVQ^w(M5Y0(O_4)hwZ4(5@-6(?js?w{HMm&j zql2Ti7Dc{LVoX549hU1r;*6rS4{47WaJLUV8a5A)>?|)qnT{q&Dqou}J zRgHCrMG_951VbJ?n5d$N5>Tb_yMJ3a6o)%LZe|(O&0L24mkg*mU(7ZA)5UiIIu%U! zMIS2&HBI-jW0XsVX^rmGD)!2p9>l_=8eLmNCb?D(!T<%Kg8gW~bOdO4^3Bqmrdsil zXu@9%P-CHAS`7ae&ddKbmJdBr@8Gsy8{3#!d8zF;A>kGwU8Y)Dw!R zU=_2hB$P4}Be~pazd5c{g8UMXKabskFkW@DQ9G(b$4r<0G zHjx=2TMR7-{rw;du4NZWd$R{b7sf|~W;-Rzro2x-IVqz(5Kf8T7jN6cUKjrE=&Sd+ zF@C2>RGSjqUu%~k*%BrzIJhnv5C+FQho?c8LCtXt_p+DW9$sJG??Qw`I)C(01ChG6 z38@tUi;}v|BT-}v@)GQdiO}ns_Fr}rmJl36uS*RPh^hy&FsvN7E04?9fGDwLrf^w$_lGf*_=6!Ru+t}hCP{rz4 z1(N+D_6t7un`wWcUXMp_(j6(DHjbOb2F0!6{zBe!oM9GuX~ZV7BQHpjfEAPEp?bX2 zqMCJs=(a0gzJut7hv!7H9t91*%QO)CL1Y&0>_)Z1@+yLlWVfR-C-v}`AL#Z%5qRBE z1M&|1j~O{sVAKFwG6)L8MDpkX|C4P?g?W=n!&H0qMP$NbkJ|2ukmvmq3Wr z0HFsGN+9>#x%1}zx$noDci*1bd(YZG_OG?R^{sE_b2Zfb>&{>QYyHprf9~f>|99+< zJjf;Y*Pm-AEaaJT()GOmi);7+r^t);e>q=Ueg7PhIhg7OZjp!UiOUw9F-aNU#UE3LXg2!Ejfr{_tEu2HPaK8hf+JnF2 zTjan9JyKPE zZXHh-?A0!XuZFG7?SibnG>p2NZbQqpg z>JGVw9TR^$%o``Ts7~kN$%^s3wcL?W{)LF1FpoLd`;serQ*|vnAF%E_pF2TJ>2x;o z0{{?_x3{<6%{fAE3>9@_&ukZJMCWH3gNz`gHP&sJRfJ(y#$cbZIWc$it^PY~zTl0l zV@y|jcKvE^(@e+p(Hebuwz$3#;ZeiuTwgUV{X;H}I`90CL%!WkRo0xT%wm~CYf^c3 zqV*NLgKD=XFm}q1&FQhxysQlt*zYo9|fk_55YrLnzns zm2FV1*;0Tysb@^jI4Vu9ue218hDAZx@1dNlZJcv}NM1dIhSriC%{O1fSg9&`)=XQw z2lUnnBHOsj%)tYZRDT&Cx7Jdmp04fzzKt$D#lM2bKSbb1iU04CmnV>v*6jY}QdX{ey&QLqbqa>cO1Pw{&;}3xzW}U~vX|jMkf->L1D^TTF|qBiX2~Z!a$at~I*$wYnYG4A(5E zjlCn-H@k+I$F&9YpV+URjUiH~pIlc#^S(zG*uvQojyXA!Iuvd54D~&Boh1w2>DN{o z5w&YH>^HJkD;ybOEgTkOkpEotbLSG^z77So%|S|cD>RdmaCYr`9dvaANBf}Bo0Uwb zGap?oR9e-2NN&$FT0~4v--s%ev;_6}%>>Zg9JJGLlMS#zpux7lHp%wXe!{(R(|DgY!ZymbV#{l^36UeekYqpH zN^M+XYG+(iXLgy1qce%(7ZK=Bdp2s22287z2i@w9ilr^3Nu~*pnvFK62^vr4-}&ng zTQpNfw1LHGRkV4;lTgTAT3rCX&MS?^vg?M1Bc+j1955;$u64nO>$nXkc7D&kEg}9d zRQ=zb|3coo=$iSm+_WwaE)K?hX%S-Iol0h-Q*kNevfedI=MC8j=BJg;Y-5?|9i5t{ zN0;EfbE7He3eBI%1|(r*2Ww>|yr~gv(mZ=V1#+WvH|)zg7l#);E=^d{g>`285mY>G znpcy%iPrh#BO7?n<>6L(v@(E`9{Ty-rO#<6y@iwmT{Cc$TZqefl#=UQ;t&3Wb zm5{6>0QZC_BO{hWB17KpAKmwPyp-iH$BQI%o;or;2fvnGl~=Z6Apl2Nn4Eq@Z^aui zvEXetgID6n_d9;vC6s;R&D``02IRdY; z?m30y65K~4jlEBaEzEmb>G=5mH*Z7Ur6|S{PLd;U?Az?2(OizE)uquFxt-s?gH1wi zp3%|Hl!2YuXs7Lqf`C8O8 zGCdv*O(*LKI{*n?3#R40g}ffnh_>XXjmLNpja}YB9TxnlVn}T&z-2?@pkbIyt|Ab`aDJM@Fj*s1l6gS_LlXTA^8ZT2BB(%v|V&YZUH;|LE zJ#vnJwfzdsFqY(9*yA=^vN@?3z2bWeq#MA%y~YsLvB@$E&8Z`1Yma0;vMDV<@C31G5lYYk%Xk@ zzf|$KQ!^XacTI==y^sHQS^WRw+n;ssLn{{}D(wEQJ?f}_;(2T1q4##_rg;Y}C@@g_ zj6f)@+=}t6XH0S`6729TuFuOubzEIj5#C}>LRzN#WH<-*c5-;~y?JxaC)029L%W0X+8ssDc?rqSjA5T%*>qTn%?l-~$7N}5dawXQvWyK< zw}Zcms%Rx*vf|JBOK?rwb8Daq~a*mx?b{cw(&6_E{9^e=k~Bpj4XJg^e{wmBiUx0b`@|wPYH;cP zdxkKS51;IuPe?qX)_;$93uA=V4q$w*WmqL_`A*tUhhBil(vp>fwYB2ox@XYgj{*84 zqh*h`Yig|LQU>THYjvME#T^(rI;SAp;>Qgtn`W!=IV_?7fxT z4`zpM8Rwze5<8K%dCjPfg)8o6N3~`|_cTT#^P|Lu*RS(xt)wDbKwLs7^r)|z%S^PNypb@z1x}!30`j^6j(H}dVcWRH&!;iqe|AfSYEw*s-5|A(359K3H+s6;1bKmL^Rqb9zCm0NIrts-= z>+^0>X9?TO_RR)bYW6q z7Bpm!d>);XbqP?@Fj8A>|9||TfXO5oIP}Vfrs2^R84Xz~nPk{DxxpiWr@CQx70wkX z6h7p0=4&ZbE0~7m(IVfR6?`h4tWa`uF!cP^@UIq)FskP(fuW{w-@LjGqo!Q zbIcQ;*S9xfaP(Ohtd#EMRF$l_Qu$*$K2=&%`@^$afkzvtuFY zt$Us{ou*v)!xUo@QT2;~Arx#95;}~o^AJ{Ai65r|`@}1V&1YEt24e$4{iNf7j}L3N z>=MS)0;?5-y7Fc9`x*B6x%iMGjvOE~;D(GxS860zJJ{_-{_XA(8siUeJq@mtB|%3W5?gTVVR=RoTWe?gjjJ5j=#bo{Bx1>UfxaQ7 zHeaCo@DchAoKP(R0g<32Ouj_dFzNEs_`O^PKjQJ&20_5A_23-8r2%hsMKXY~3oOqe z-Dd6=p5yN7N7fT;CD2#_sZ!ENZy4lib$8-=PX6ywIlXJ=E^qMelpY|hFC?IPb>^3N z*OaIoUePDlZy`k51fMLgDce#nx^*e-g{R=GHtsmp;KRCIk0f-3DWT^MSz?!n2*XA{ zxnX&=?sI|0H~zu=%U86~+bjVu&n`}etw;gL^hLiolmF0Q7Z?aZv-$@$%b&5<0wGUA{Wpl?F>>A$wG-`Q%iU%-1k0P*gp;_vfI`AljDQ6d{D zrdzB$)85$B+Duigoq0B4GTaSWENH25W~8*IiMxA?n=e*z%xy>148&;$LACIA z2`s}xr;8Sf(kVdNdW@!kjUid2alr<`n+J7dlbUw~0=n5qg_z~`oS7RSyQ(n@k8?4* ziq~09^0{u4H}=9aw$mAJ+Zy#4f`^II*6KGtX}N0T%oJ5Gcw9~!?wSw1g9ZsoGM}fV z7o{)y6rad9l2(f?r3GhpBEA+d z!VTeeV{f!rC^z<9?Z~aUE<;ac%#6Kea{huBjndjI6m>VT9bjS*1nr_HyK{8g=RAiJ zXf$h>y_u7Qziz8H8nw#u(YHp&N8>K(cWg>3n3~1vr`D%$Sv%`JFezcVEk9B9KRts= zUjkMYshPa_uJA{=wRcu$ayq`tT>`a8%d$ z_3oyA7!>AoMf^hxr;d-HujNTg;`v4J85@$?5Q&e)yJD>Bt?lfDlgX!YEp?qz!lG0n z;m61L`>A2#AWBf3!@j;P2S{c-zMxclyf)`(oNR(1d~Ii&+@Q4kDP;uP2@|+V`y*N` zT)2Dkb;g{C&Ag|PDx?LA8~Y>UZa5=TYp}X$QDOX5>hg}Ca#TtqH^a`rUy&-&F>=GjMx;N}q>4&xW+e;=4q=kALS zA_7w>vFs!KQb8YcP*rui_aj<(uXc6gSK(@s&K6bryr^yY400QAwuX?*IBODov_KH7 zZ>tA-jPMd#(RI?2JX3Us9JNc8&Ct_ob*|Kj?6;T>SIp|(~SE%Mkg$ojyOI< z?e>1F4`s(#-IJH1YigCo)2FF1IJx{39dMcUMQLnfU1lLZt5yFRG!J>FH<)`Obd13-ROt%}xs!CG+OqYy&R+$<2T+KhqWOj5^y%3mhTk5TKy808<_L+P}LNbTJ z4LoVaZ7$Nr6$0p==(F-3-%shvjia#+B2&~>&h7#4pW`PoY%FS20o$#mYwEg^mq@@GtM2~%-Q`I&%80&4r4!} zlFDPb2VCp}mDGRj8wcADr0&qu&y zqxD*uGARFFmGn`|)V<6oCTatwMh(5>pzg9dzU1DqdXAaBnztGp$v8cj?$%w*b-ryL zEk<<@N>%k~B}yrEyNMUp749e)fL9r@Ko&Le0C*8Ry&(@Sb1l1PxZ{(@PhU%N$cv?H zf|rsQj9+&-FIl7Lhv5Cp#(i>icKlJTECfCnh)CT ze*Ut5{zYotO4BFx!?I68N}Si*bNfv`C$H4Iqo4uu%(Y$bE%KY%((5IvG8-CyXvs3J z+0q@RYzeI{@GpgGYlQWU)_m5laXp=ldEm&&_clV8EJ@ML7xzl)%_a|9_=a`1MTlFG zsOk5sfR%aCC5ts1ZEX0#_8`|t_|I|a4-;gaw(n9_GeN3^tS^~jip|-ahus=+OCmX@ zAnve0Qr{ojRV$3v*nS>57uTs z2^k%XFw=cmp0fFRoXa3FEkR%RUwuTg$Wpf1QsQ|=n0ewyza#&p&TJE&z_~wEW~Dt> z4ws)hQI{+2(fZvp>&<4HSCPo``p^^moz=FXOC98;Kd@F;vNr1UB5gV6_N%S^ps!=H zqaHH&rld$KX?!>9BFA&4OD_q-w3?6?5GA`fbT?Cg-$b0dSwsboj|pVGcjJro>7?@0 z(VVg1-n4ZQch#=UILiqEP22IPTqUl>k}Y)U#biUR0#0$Uo!|G@o{^RRjq+txu?V-d zCqK6_QTa|+Ic7JdEX`vA$&zd-Z{CYY?7O~*y-(mp6GVA7w#37>_7s`s>+aH8_<84@ zz6U8kTW-BA^P>T@V;$1idoy)EG5GRjRvPDd?JSUc$2I5q5|U#D4QlW)IvAerDoyh) zWOBZn*-dnmL7x73XwJgo4#qIx*4?&Z@6$KYyc+y|)lVGWAWxRx!2IA#+~a2T;Y6Hm z?K7sury{Oa?E{M9z^D+{gsq0j^}^*WcVPxXM@pJqscf$-U}^E<9;h?iyvf&BlXtPb9o3gZ`v{`}axkjePXLFNZvKyP;qtB~HDCg@pJ^Lqr z7hawhqRkdR1!R~0w21!DWt8YOZ4yJ}uAfD-{kAn`Z$~u=kM+uuEkquHY$VT3&DM(I z$hNlm#lddzwy;?(zB-Xgy1gI)+ihUlZVPL4VT)I8os5QI*j8Gm=W^||3#0>6GQw+{ z*MNM^^v*MGu=RLo9V=kl8pJ%?0FHa>%f0DGnC| zTg#1f)EmTN+eLigd`>j-7TjQ)nWThgTs8`@o6c#S#d$^}@U+9*1v)IFNr$V@ z*+T<-Vt<9*iHLiw#k=D4f%SN+dSMf4!51jag3HVR)w!Zwd+R)BFERhvNJw< zNR-4|l*zM8Gn-7g%;1o&>B^-X4$`V~X8HNb7iZ;ihAf02)wN7sFr0^&O{T%1WOqgK zCiQf@FO~__*?aDtUVR!f9&Ahlg3BW0;9^yW1(u^vM^2{_vMJD{ zXL93YyN}Yqc30f~d$?%Dn{Pf5p|i4@~sAVQH%Q4hbR$jNNv0dY;&hI;(Fj?l-{Sadj2J0k^26cX{$4B1c{->{oWkm z`glR9_TBW&T$FTkj!HEfVldFHc(t=IUI{I^;2k`7-WtRje-WVpOH~gPHq-E=-v{yX zUpBrJ-d%-JM9KzL^a_UCCdOQ_%Kw3M&#K$`|z7s|jipM*l6?x<(g)6{M z1G+99!qyF@vmW;+$yRfe5{Q+z9TFjPX@$Z4`uRo8MGqP%PTPzS-EL_7YCg{KjvdtW zH*@fcTCw;P8{-_UchgiVHDXdx#h;PfRjNm&oGRXIO~9gUz}lsK+}cIM)iOEN)Ma%# zKVw;fPoun;iu*EncK-0dk}Z8dTQ+y_pcj59-+RWaJ6yjapi0hXws*?yFk#Q z*CwT=O}w(^xmi4!3ZDJP!&E))__hLAMTsAGGY&F?1?qS7CvLGe+xjY^JWW#F9rcu( z8AKuW5-Y(nA%YBjU1F&%M7ebWZFj9gPmrMOL)(XsMP^#)P1_#&s8F~@u-tC)8Fi&)c%m}#0!>F-J1ccmUJ-acNxuOBA085O4nV3ys7nfEG(yKow~mHeYBHk;<=8OHB00V{B5hE zT*WYlXB?lKe%inr92c0VU8!RqkMHk-27ezAU5n4>b_EqTs1N`1+I{S?_L}SaE^BR@ zvUpc%@zM`4DX5kIlFp%q*BxmO;*aA7Fax-2QX>rt3uAGTyxVYNQ` z>uv9UEq|6s5GxetY9-dWm*p){w6bNRp8SNRvoOqU&ra{fF(6SX3VsTMFB`5Ys!P#L zZ$kuEwiEa7p9_pwRgb6hmY?%O8ijWgPS#Qfn|#Uojw)-<%>gxQqwDf8TJsX}m`P4q5h?vp^3s!Mhy$iqFE|P|Xe-LsPlJho2 zGduBZq`mH_SIn|@@tho4Tr@Ne9@C`NB!M!la%@nR#&2JGZz=3%DLh13X6s?;&c-Lb zVrkoxmi^%pZsILkYTK7`|47rWZnu{R21sGmC9O)q**Qst(RdrC`7pm-moxw8limg^Yog>9@AQyG^7Y86E!8lPo`r9%vo_olZ|& z_2TNA^N#p!B<-Z*(@w@*^T;?jJ{4~cQ)ViWYwvDSkjXB52b>Myq+T)6Y8Nh2BhlsX zh0(!$OICNYMQy?$t+ne)#e)!YXEwm)sRx<+)WdjOQ`VYjh{S4;Oo(}pv11a~*nSW% zMlI^XIVPSEoKioOP2*Nfjq;R|d*HHXE6GF?NPB?#DZ{ZHR+Vjhn=vWubicL-{CR5O zA?nmMf3}4qPdR-pA-`yf=25AyH}@#D;adq!j{nrUocI=&7=jsmHJTzr8|SGyKaigU zS8teqt4CSn`UU@WgIy&F9AziH+-e-dT{(PPPncvjg5c^CpZT%~7C&8T1JVgC)z&!Z zi1Q7M*M}yvp^8+Mg15Cw#7>*@Q?yn(F1`2VZcqrLt6-Dl~99!+(%< zpR%&v;pwQ4RdJ0$K3T&_3Vs`LYMM09sEP%aX9HQK&%m(Oi_Q#WkM+#4&U$lwC=~#A zTOu)Vy#StFAoH^uDjCsf_~5qU_UO3u>*6nw(xQVCf>?}?BVvx=E-}(DcB&j^eD-iw zdEhPR-Rwi=r0VxRu$c8dgQ)T!OPA7R&l96lgSdS|+9MGk$CuBu#d%=@4KjGo!0AV^ zkN%UHjl$A+r)kA~eE-z^tFVEwD>l#kBr5XLJ(nG4pB`{J_flmZ^k#Xrx<)N1jaQ_q zQ>A`omemgGaS1Ut;=#J5u{xiqiAc1|Jw`OBqvx87r+=hYz^LUUkcw!(Og#M?^x+aq zxjrpl<6OtAnPk9q7(fcvviYxQu^MpqD*gvNmYA1SvAS&)C+YOMr-U_rrJ|0n=t*j| zItWuHdsb!94f>k%@fO{IdVv{A`^$MaJ$iWi+8TCnj_Lo5{W@?UE@{px2q+1<`X-`# z*mo~p|M>bEBMJpjHu7zMpGVL`URd7dNxbVo75i3;OO=NaitO9%=OMon1??rjdFO1+ zFCCUG5E}TH^DFl_m>}%^*rl@6IR#r=UEqe|{qQR9G{~!zm$iZu;;Zs#9v1k@| zG!l)~*gJ~084@3_n7U8J;{P&<7c&;Qe6jIn@c4Gdd?k%ZLCRF;CilFOJ$2QuDs|O& zp`gZ|Lu9E=sWJb(gjFx->LpW~7|szPP_^{Qy~*I?NZEH`-cTQ~NRoS<4ukcEExpRf z56jxtgEGOWzga9WDC*Xcp{Su+*ZU>f)@iihsIAcKS1UErdt6Gd_biv<)~}>Dc{Yo6 z{4e$U z=U-r$bK@uRJaBpX=>7-ca7FuPqzN|UZG7I=G^bingm*apumYXgJ^2Fjz_>;`2%1QP zM7sE9Y^qI^=WkUSuU<^@sSSXclBV$@4;x+eV!MdrLoEiQX6^#^Ef*~Hu5-{={76~< z*A1!4{<-tD(#9QF)gfh2?{5*)><;$YjLjRjtjqM;F9F*Lig@bQ_Oq8260oD$c{|7f zwMu1>GyB|5nE^=O?b~WswWl+#_SL8ng|fdOSghwr`gS$b?)ca!5aVCnYq;<|WIpH1 z5OaE#%`&Okz-8d;C(l3IE?pbEVl!J5LW7j4S`yv}Y_0-AdjUUIM_aR6F9m2x_8E1s zxcC|LFt#~Kp!dN0z`^_M0n3tZT9El4``D__h6qc;Mg~2FAPV61QKzd4Zn~)an#>3@ z6x`YA`-!)czo}5NQMk9gFi~>89BW8ZDV;^dIE_aGC8%?I=J-_#DodyK`Zx&RGY`N~ zhL?W`?9zre{0(nB_?=k->IC&W@y3SlbUV{%h(;2P=gKh_*7pPGZ_anWC0=AaXZq34 zT66-*A45a8EC!ha=c?U4p++V6A-3(qSc@;GZZEnNoLpuF?D@``iH-k>v6_BGKX&t8 zpaAQRkq!Huez70yF7Y1WgjAE^`h2{SQ+Il0h_>Z9-g3LsY;G)M%>x*a^>R=Rp+m{| zRNC6~-tZ;^nxs@E@t*IkD=BPovg4{fxr_~2_Mj&Bf%}H9^LxwR1@M^@z+M~RSvTj} ziq9j=g!U$eu@%b=j#EYOFtp2wKZ&IPZx3$#ry`A}0o_xlH(yCs|4>qy@fYW{Ny%id zu6u7@DOcT)lR3M+Rc{82+0p|zLSr@YBcZO&jE*ff_h%FYw?!LM=aIkWe@&}h-2l6$ zU9c9-Zv_Uye0=neaX^#qoPyb*4ZsxLiQj@-8LI9yHB}K*92;0FWRpOp>_VLyR^OFq zwv_Ok$zRS+?~>_66b2=`kSSVX<&dAT*;}3(wU(413fGDL^ovPn^8w(s5ps69NxZ3A zQ>IaDo*d`YBeT$g>2k{ltIaL2Bn59~tpz!*CrYdw9vgTS!JOx0pW0bnB^Q>* zPfoK7gdqYf^{I<-VFL#rIARTHl=90P0^UR)%C(BW&>BsI98wyi?sS2Zh~|W-k+ffm zBecbH=UWK{Y3;hb_d}lpYk^RTS>A0c#(TYf1US=TWlN>YNcfm-3M6jyS&~b6!)lju zbz7nqg7T#G5g;eIrF5!l^^IJ&!-UNYuIInbb+0ppCg138Qf#_#(ouEY;PUo}%%{wr zQ}Z>6v4_A^;R%?S2eeYEuiDwOwcR`DtGCRPTh~9bdo-lt5KFL_m-|n?(=5p_B5ZGx zc9gSJ+mOcZ-vP^Wik8J0R(~U~A*xoc;_!#4eOkL_mi*D*7L9Ur@5)*}-GIt&O}2E- zy`rqMow2Ls7NtpQU~iWyS*sy{)Y~@@U8C^Ms*4xb@jpbj;6nqOx=1u@fcY)>J7%?3 zt_rT`;~~h%UiF!nPdVzP)X*y3b!YV};$sEw+`PQJRj($39U+xA^^fy@Ap1F!H_`nm z*z}72di-UnlQq3v=@R#1w*{A|>+33yhaC;G|{(%7o& z_{he-xvc7iq$o%9y!Qz#O(tx?jj-oQrN z*kI&{YchVwE3H#=Z3EL~4?Jv2kZv@t9;GUTPsQFKYC>Mu!c9H$BhQnZ&0BRtI>X_`I|ayg<>!#Of3li z^Xa?RO8Z1%@u29QQw>FS-@4<|A-aavfW_2nF=;dEWWgksvDLlAmQLC=4xDc3h`c9i z^1RUi4H{n=9o@PG1NjJ-tc9H%NrJenL-zP|M^-`IlF?D&K|8w2&H;2-fa%J6*ORxh z9I67{amNl-ar`>EUUoR4@CmaI*eeR4uBWtYrK&(Sy96(BOqlm1!O$u*m>X1MR<+99 zuC~^QJ-3g#w?u7Z(U5xz;X1P&=|zlbyc5Z5g)Gvv^PFk{yF&xo;6J&RPkp{fB_)B79Qg8iOwQ!sw&UfoZ)f?lZetiF% zOxAwY+}`D#xn#w27seX`x3?v4-xcq^g=tB=o&d28tfTM0J=>pZLw%%;EtCw>=XT9k+V)$M#TeH} z2KyZ3Hn*QnntjfJ@LE;BcZ``SGaw6^@7kN^nq%w&|{#i%AF8gpr}a=S=1aa-O*< zK2><-clk0Spswey*6Jj~|2Ut>J;~LsW@7Mz>I0Vw81Z^^DV5>na>a$c@otLI^aKaN z&3WE2i1HF<6=9-*J80QrSWjL!!-{Bb;H67XeN4i$wU*~Els{(Sv8w7@rj@2<-IO>+ zeBEP$e}HLW*P&73pV{a82Sdr@?vXy!mg&dMpVsL!UX@M|#m`G0UGAEvjY9SqgI>fM zo{XiP)j#DM7Wxn}HeRzMf2Ow(WL~_9>Y3^2jBmGNYx`!G)4$f8fwrwRXfc6k;Lx15 zm8109Slj!>599B*OuKY_-98uc$_}!|?|h{SENY)NKurGw#0I>c%wI8^=!pn4`jJW6 z-Jfc&!Yr+3r!36cnFysYSB#r6CV1(t6rbv5^$+H@3~%?asf3uFz%M`83Ra$b&omOm zvw+>Gj+_w(5F3RHIe+Wv(lFB>Pl42~6!&BHJa_VOHO{Q$#Zb1hxb$@M^EW}f4eS90 zX6ixlD#)(*=?m7qn?iB6hHru_OIC@@0uyN?N}(sG9|_wd5@m8wB&)v$SQk|)S0Go- zuY1v9H2M_wO(65s4%2y>4Ko{|HwrX9Dl=sDl(tdyyzgwJ%l#_6oMx<82CL?oOJ>05 zt^*9G1=Y`51*^KJOBJcOQ)Xb~j?~HI(HiW%`4U0#_w%^A;b>DP2hUbrGj6XqD?;M? zYM8*W&9=bE9iKvRKXppE=si?gML(~5W{{ZW)(=H-I=_e9;`z18-Xg8sTTgjPjkhbA zyT)*(F!YoiA!3d_v3OPGZYKp2l}&S_C@%nQ@_Rm)ff7lvP9zME`HrNkpO|T$R88;s zm%9J{ni^U?VlJxDc#m*STdJMnUTU1Tmkq#%UectkcN^>7qvF=Bf2pqx$lbDtDii^1 z8`?oLxf+j`yDYZK#LC)6fD5gG8hos; zZcxEA2wm(m;Z;10q<1I?KEek!r%n_$O*@vPgdSIha?M)!q&wA?{j~6cN(or)C$J_) zjdKjp?&d#^is7?zs5U&)1J>$`=Z;)x)LP8n9|Ki+bzlYa$;9LB>aYGe4?lKI^x}xG z@h@Gt7)l-~8q&d9?WLC=_VXTj77HJF@?S0M$-HUB_Z#uH`hed*y?mfl1P93Dy5M&l zZOj{Vs|2l^=53~o$Bqtu0uSxB;^&#?3@p@T`<^C50WZ*j5;k35y42JRl9`D-jlyGI zDdq3jcN=Z?P7kfYb}FW6$tGSwHuF80d&%rH>PhSeS>mm5>+-oB zW6h~4d02S*o@TMw#!lyPsLYa?zPk1CIop;_P&X|@I+SNVYQq7%v%j1ut^?1uYQ?e1(YrHr zekub=FVwN@qp+EjTl0cNT-C++i_X*qM~z+9kZ(<=I1a>E^@p$9;JDqlFB{ zzP0@9*^kn#GPO&!YvBzi{yzTuNN0%?0-Z0?RqH4q=T;9VP})Z{EA#laveplmMq}SO?>fG z+V9ZJ8v`D32>43;+#Y6Vv$0h=qH@sgndErXWmrBo^ zA#~IvoQuRf9IIv6&a#|GT>OX=I3-0>ABqkMMJTayEIZ0fVoo!LZMuSzzPRVflvrf( zj7xS6at=ay$h|MU(xiJ66fIRSoNRR|C1t65_x7;*qdP01Z@@LT|K`sgL$6k5%z3;d zNOH0V4A9z4ES_juP)( zR|)R$7Wa-LJ@^LiJ5hrWT_MZ!0xJh-si^G~(ff~R?6CKdbZ*mpxq(0= zVf(2W?GlmOFMI9M2qel9NjHNi1935<+lw4nbRKQIja<2^skeZuXhSDjTK>h9?CPO~ zjXCvDfrTorpkY{c(8O-gUPXplIR={%#X6kx=su6OTU}UioM! zB|D%+fYls^$o^X1__=35`9SI}IgOmccpg#eQaDT*yTTrJ>Mh9*A%?zMA|iurzro&o zeiNW11{j+F$o`?w=Pn;b-Emw$AAxOcKf`?-c|IZgYO2|HiP(^B_tLsjqVD}8-srhn z<#n-+k=lK@ziVlre30w6uE^#S7;U9(qD-dBW#WwTL9B7Wcaxqs!vbzNF?V19wZ%@Z zY=HVLvBn>~L1`wCP0CizBAL;tX16H|6kq95uFxD;9R6yGobEGe_^DZWFi1B$VA`lE zC-EfO_T#{@&)c~St>)N7iue#6(aFsoDwmh6?rYT8@FKcAKRCOLiRSv#g?N?L4ii3^p6Cm4^4 z{_69Jd5~PxWxo$lrgZRh=^PJhQfwdGRf?JL^Hk7qmPpHYIuhChVPmbmiM? z9WL&s5vJm_g%_>BAsTnWK~(4cnB@$g3gRoEpL9Tmh8#R#ezE1fqb?ELE=;)ev!~xb4?!gEW2s|VY$%p=E`Fbq4mD(=a{bQOPc$vtL;~)B=xrmAZK$z@ zr=_R{InlG*;isSPf6Kpn-aF*pd_*H2=pzitqHm1quG9+7`vbi!GGN>iU3%uQlGmyp zBp5g%z!i~Kn0ofg#9DrVzKn^N&~9gL8;)xqplT-6*C!LNJq44GT-|0@U+#;=A=6=R z$FtW~<};Wwco2!8GWf%X5qYYQ$ungo5*pPppQk$Y0OBOi9TaD3?}2L9uQ)zHz1d8@ zSgWn+!fWH7bhNLx=Z1MiDVa9;7mDh%vr7lR555^1V($3$K_px#1(gl=@Z>~8pNLa4 zMy)RpC2bO#p&$OqWHNEIDpIc2ZV4EBJedy{TKw zvPe|th%0&K6H3hsWy@?XeC>Hw`lPzdVb%J6X)k}!4yY;dRF`)0#<|ha^_}v>tN(!2 zd)K$KZd@V{_Mes!6nq&31LlM74K%uhzd2iBH2m}|O1^S?N^h?{R4~u8O6cSW@_5Ox zu}lH*MJb|1%s%rJe}w3?*UqsYbv6n6C6GLJ zooBthE^oTKw%%Ow26>+Z$eUlI7ccr$TDh#a*cA`@PSq8XzAXs|inf;=rIu)_1@_o3 z&Zr{ThILy+=_Lokz{C67{|$o{Y2e1C!;nw=@(R}-eIXJDxTEHA z?KFkqVRt3|hOOc&thY2!)HPvAQxM24ca>Woitd}X^lHy;tXdDsA@_^JGd7*n7_zc8 zJe=+{U0mgexI0=d_TPYY5Tmnwq~s>uk520|HOUIh#v&3gj1JQ|Y|E%j${=w`TmPEE z`HM!Iy)`7I+IvzN1wk}|Q=Aw*$F$%14c?&vp|?ZNA{4iv=3ozt03t3S(1~!;U@HFo z@-i@SuqgFnu_9m=v`V$Kb$2h5zk+Q){?`PpaisL zhOy?uCX%f+Rz_zLV@(a6;px;`tVHXNbZu`ubXWWoZB2ab{6%viG6(=F65eR2YlL66X z(Y`)ty0j_#AlP=g>~6Wnsfod=MrEHp^$F$)k4NT0Aeem|e&u+e?)lm3b6!u-fwcWa zkT5(wVE%=)<1H%$FBkZADAm2_MiFVRJUtfgp+?qjk~}t5Zv&2OqbtKhlz$AK zrL^H&dyl9t+tdk=9*k4uE?P=Lc=bOw;XkCvJjq#{Zj(Ku%7s^;@DxT{=JSHx=vZBT z$`(jit!eD@*#ELtlK!>(TO{y8PN*>0hj2onM+kz|hvYTyFNL-7&y$hp!Le>?~*J=m=mm%CNC6pEjX? zdw~9bu%0w+9scvbcTtj_h9Ezgqqv5WypbHapW2}*zox^UyikejYixTI)`NV47MN1}*Q~ASZnSW4CAb&rZSA)AhvwAiHaG-yv z#9m?HsM&z8bBT)y+`u{H0-HpGdWI?2W6_fVvGNdg{mhD_F0m~euSb<9{>EC5(6Ae^ zyl34|At(YN;|4A7+b9Qh$j;>@B{{VYcC{7t>1qAOIe!rCCl{xPBo%gw4f=^R#}oAf zAgcODhloN*LVQV^zAlB4xgSiS0;uM=35kt2V~5IcZ04Ds`$x?^YmLCS$8*R*+Joxj zb82c%0#*vF51#)12=cE;szbuC~@InZCy4$5!G_ zq-Hf-7s=#~R}WxF{k<~hQ^_j4LN~Sq5lq;M#Rbaomqu4a&g30V7wV(Qv#qXPSy_F} zZgsexAtY7pj4AoQR42>*d1qrhg!hNj&lk?6#IBn$@3vBO>oR)gh;UPLeamTyN`P`Y z;iO;&71w^*+)T?|^L<44&r6Y8sd!FZ=dWM2#_cSfgG`Sz?X$-aqS;H0-bx*kHFmH+ zAvli^nB1cB`Y<3`)TzT<4wQ-j04tyIX+AYYzv)ICi9CqqUknV@G7WRo@TcjV6h}I8 z$o^`A3dUflTEsJ#^BgI~WtX*;nv%&8x_S2N)4`K9V&s^t=QgY-H%X7BlTWFnPdwOo zUYiOc=Sijy=@pOTIo@!D+(Jf40FPBK9&pmpboT(!{gUa1WDg~g#7U$}45@pV1PQ$) zflf$J|67vi?ZdDv)#^uAMj0dlyO+YK>6{m!71AHBaWGVrMEW-7nw~?t48U;E8C>^F<`Yvzfq;U%$@Gv|&c zXcb#64UI-xVcjn;0l4)35u!;b;Zvw$7%uuqtt`f;!n;`wF+#NL#`r=Q8b^{v$w!F9 zKG|T?3f(}*rKI3b!cotyG3$-ot8$IS|WwLVBw=62!q>+Xdc>q*1K zc}UdjT6n^gJZ1ln0`XC{BO?Mo$xp)84nPwf1T*W*8c~57;G=7^ zk&MI}8t)qAobJ@=j{XoiS3S9~XQN#yX*Qp7(*S+~3kP&&%dhvFIf~@dPIgB(&MhGy zGNV+p1x5nQFHS{WyR~Gg--3{NiQ8*RwOm*s zVB1ArVRZTbjYDfK0JqTy+vOQJ>(mN|x<1yF&n20qpUm9cPGKrjr787d$Ocx1}8 zmqF(WMGkzdx1FN%)kQ1X<))I;wkb<<1U;waxK0cBrW!IuAX4jh9c$jfr8`)+KQLrF z`#d+lKKE40f3&cX^FQ#?g86u%+Pn_+hdMTr>5paRbT5CfvO;QJU-|K#1&p=b}&^hZ-GMB{p|KiIm1XZ_)) zJDAoVm;T_khco<<)*i|HgV7(&_Q-}mmaR}R%VQ<&K|A?)I zL;jfihrk*h^G9kmGV>3d{h?yl>i^hy2kZQ!&p%Y&>AL-awMH=h(DaX@HKOr{uYcsN z5#Ihd`iI%RX6pj=cZbS9+TQ7{{!n|TTmC`&55aeb`#%cb>7V~7{fA{;Bl`yhxkLIN zwf{)%KbZf~`j1@w0r`*WespriteLayer[outX] = palette[tileData] | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ } @@ -97,7 +97,7 @@ unsigned color = (renderer->row[outX] & FLAG_OBJWIN) ? objwinPalette[tileData] : palette[tileData]; \ renderer->spriteLayer[outX] = color | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ } @@ -119,7 +119,7 @@ if (tileData) { \ renderer->spriteLayer[outX] = palette[tileData] | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ } @@ -132,7 +132,7 @@ unsigned color = (renderer->row[outX] & FLAG_OBJWIN) ? objwinPalette[tileData] : palette[tileData]; \ renderer->spriteLayer[outX] = color | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ } From 83673cc5210aa50bcacb3bb5821d2974d6a8ea0c Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 11 Jan 2020 14:02:10 -0800 Subject: [PATCH 06/20] Qt: Release held actions if they get rebound --- CHANGES | 1 + src/platform/qt/ShortcutController.cpp | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 08af9bcff..223b90ad1 100644 --- a/CHANGES +++ b/CHANGES @@ -110,6 +110,7 @@ Other fixes: - Qt: Fix undesired screen filtering when paused (fixes mgba.io/i/1602) - Qt: Fix sprite view using wrong base address (fixes mgba.io/i/1603) - Qt: Fix inability to clear default keybindings + - Qt: Release held actions if they get rebound - Vita: Fix analog controls (fixes mgba.io/i/1554) Misc: - GB Memory: Support manual SRAM editing (fixes mgba.io/i/1580) diff --git a/src/platform/qt/ShortcutController.cpp b/src/platform/qt/ShortcutController.cpp index 2b37fc014..bbb89255e 100644 --- a/src/platform/qt/ShortcutController.cpp +++ b/src/platform/qt/ShortcutController.cpp @@ -45,8 +45,11 @@ void ShortcutController::updateKey(const QString& name, int keySequence) { void ShortcutController::updateKey(std::shared_ptr item, int keySequence) { int oldShortcut = item->shortcut(); - if (m_actions->isHeld(item->name())) { + if (oldShortcut != keySequence && m_actions->isHeld(item->name())) { if (oldShortcut > 0) { + if (item->action() && item->action()->booleanAction()) { + item->action()->booleanAction()(false); + } m_heldKeys.take(oldShortcut); } if (keySequence > 0) { From bd87038c9b611659690f7927990199548bf8730a Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 11 Jan 2020 16:05:08 -0800 Subject: [PATCH 07/20] Libretro: Reduce distance to fork --- src/platform/libretro/libretro.c | 33 +- src/platform/libretro/libretro.h | 1404 ++++++++++++----- src/platform/libretro/libretro_core_options.h | 372 +++++ .../libretro/libretro_core_options_intl.h | 209 +++ src/platform/libretro/retro_inline.h | 39 + 5 files changed, 1651 insertions(+), 406 deletions(-) create mode 100644 src/platform/libretro/libretro_core_options.h create mode 100644 src/platform/libretro/libretro_core_options_intl.h create mode 100644 src/platform/libretro/retro_inline.h diff --git a/src/platform/libretro/libretro.c b/src/platform/libretro/libretro.c index badc917b0..b73b2039c 100644 --- a/src/platform/libretro/libretro.c +++ b/src/platform/libretro/libretro.c @@ -26,6 +26,8 @@ #include #include +#include "libretro_core_options.h" + #define SAMPLES 1024 #define RUMBLE_PWM 35 @@ -49,7 +51,7 @@ static void _stopImage(struct mImageSource*); static void _requestImage(struct mImageSource*, const void** buffer, size_t* stride, enum mColorFormat* colorFormat); static struct mCore* core; -static void* outputBuffer; +static color_t* outputBuffer = NULL; static void* data; static size_t dataSize; static void* savedata; @@ -122,6 +124,13 @@ static void _reloadSettings(void) { } } + var.key = "mgba_frameskip"; + var.value = 0; + if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) { + opts.frameskip = strtol(var.value, NULL, 10); + + } + var.key = "mgba_idle_optimization"; var.value = 0; if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) { @@ -134,13 +143,6 @@ static void _reloadSettings(void) { } } - var.key = "mgba_frameskip"; - var.value = 0; - if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) { - opts.frameskip = strtol(var.value, NULL, 10); - - } - mCoreConfigLoadDefaults(&core->config, &opts); mCoreLoadConfig(core); } @@ -152,19 +154,7 @@ unsigned retro_api_version(void) { void retro_set_environment(retro_environment_t env) { environCallback = env; - struct retro_variable vars[] = { - { "mgba_solar_sensor_level", "Solar sensor level; 0|1|2|3|4|5|6|7|8|9|10" }, - { "mgba_allow_opposing_directions", "Allow opposing directional input; OFF|ON" }, - { "mgba_gb_model", "Game Boy model (requires restart); Autodetect|Game Boy|Super Game Boy|Game Boy Color|Game Boy Advance" }, - { "mgba_use_bios", "Use BIOS file if found (requires restart); ON|OFF" }, - { "mgba_skip_bios", "Skip BIOS intro (requires restart); OFF|ON" }, - { "mgba_sgb_borders", "Use Super Game Boy borders (requires restart); ON|OFF" }, - { "mgba_idle_optimization", "Idle loop removal; Remove Known|Detect and Remove|Don't Remove" }, - { "mgba_frameskip", "Frameskip; 0|1|2|3|4|5|6|7|8|9|10" }, - { 0, 0 } - }; - - environCallback(RETRO_ENVIRONMENT_SET_VARIABLES, vars); + libretro_set_core_options(environCallback); } void retro_set_video_refresh(retro_video_refresh_t video) { @@ -824,6 +814,7 @@ static void _updateCamera(const uint32_t* buffer, unsigned width, unsigned heigh if (!camData || width > camWidth || height > camHeight) { if (camData) { free(camData); + camData = NULL; } unsigned bufPitch = pitch / sizeof(*buffer); unsigned bufHeight = height; diff --git a/src/platform/libretro/libretro.h b/src/platform/libretro/libretro.h index e6809262a..d0f1042c4 100644 --- a/src/platform/libretro/libretro.h +++ b/src/platform/libretro/libretro.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2010-2016 The RetroArch team +/* Copyright (C) 2010-2018 The RetroArch team * * --------------------------------------------------------------------------------------- * The following license statement only applies to this libretro API header (libretro.h). @@ -32,7 +32,7 @@ extern "C" { #endif #ifndef __cplusplus -#if defined(_MSC_VER) && !defined(SN_TARGET_PS3) +#if defined(_MSC_VER) && _MSC_VER < 1800 && !defined(SN_TARGET_PS3) /* Hack applied for MSVC when compiling in C89 mode * as it isn't C99-compliant. */ #define bool unsigned char @@ -77,7 +77,7 @@ extern "C" { # endif #endif -/* Used for checking API/ABI mismatches that can break libretro +/* Used for checking API/ABI mismatches that can break libretro * implementations. * It is not incremented for compatible changes to the API. */ @@ -87,13 +87,13 @@ extern "C" { * Libretro's fundamental device abstractions. * * Libretro's input system consists of some standardized device types, - * such as a joypad (with/without analog), mouse, keyboard, lightgun + * such as a joypad (with/without analog), mouse, keyboard, lightgun * and a pointer. * - * The functionality of these devices are fixed, and individual cores + * The functionality of these devices are fixed, and individual cores * map their own concept of a controller to libretro's abstractions. - * This makes it possible for frontends to map the abstract types to a - * real input device, and not having to worry about binding input + * This makes it possible for frontends to map the abstract types to a + * real input device, and not having to worry about binding input * correctly to arbitrary controller layouts. */ @@ -104,43 +104,52 @@ extern "C" { /* Input disabled. */ #define RETRO_DEVICE_NONE 0 -/* The JOYPAD is called RetroPad. It is essentially a Super Nintendo - * controller, but with additional L2/R2/L3/R3 buttons, similar to a +/* The JOYPAD is called RetroPad. It is essentially a Super Nintendo + * controller, but with additional L2/R2/L3/R3 buttons, similar to a * PS1 DualShock. */ #define RETRO_DEVICE_JOYPAD 1 /* The mouse is a simple mouse, similar to Super Nintendo's mouse. * X and Y coordinates are reported relatively to last poll (poll callback). - * It is up to the libretro implementation to keep track of where the mouse + * It is up to the libretro implementation to keep track of where the mouse * pointer is supposed to be on the screen. - * The frontend must make sure not to interfere with its own hardware + * The frontend must make sure not to interfere with its own hardware * mouse pointer. */ #define RETRO_DEVICE_MOUSE 2 /* KEYBOARD device lets one poll for raw key pressed. - * It is poll based, so input callback will return with the current + * It is poll based, so input callback will return with the current * pressed state. * For event/text based keyboard input, see * RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. */ #define RETRO_DEVICE_KEYBOARD 3 -/* Lightgun X/Y coordinates are reported relatively to last poll, - * similar to mouse. */ +/* LIGHTGUN device is similar to Guncon-2 for PlayStation 2. + * It reports X/Y coordinates in screen space (similar to the pointer) + * in the range [-0x8000, 0x7fff] in both axes, with zero being center and + * -0x8000 being out of bounds. + * As well as reporting on/off screen state. It features a trigger, + * start/select buttons, auxiliary action buttons and a + * directional pad. A forced off-screen shot can be requested for + * auto-reloading function in some games. + */ #define RETRO_DEVICE_LIGHTGUN 4 /* The ANALOG device is an extension to JOYPAD (RetroPad). - * Similar to DualShock it adds two analog sticks. - * This is treated as a separate device type as it returns values in the - * full analog range of [-0x8000, 0x7fff]. Positive X axis is right. - * Positive Y axis is down. - * Only use ANALOG type when polling for analog values of the axes. + * Similar to DualShock2 it adds two analog sticks and all buttons can + * be analog. This is treated as a separate device type as it returns + * axis values in the full analog range of [-0x7fff, 0x7fff], + * although some devices may return -0x8000. + * Positive X axis is right. Positive Y axis is down. + * Buttons are returned in the range [0, 0x7fff]. + * Only use ANALOG type when polling for analog values. */ #define RETRO_DEVICE_ANALOG 5 /* Abstracts the concept of a pointing mechanism, e.g. touch. - * This allows libretro to query in absolute coordinates where on the + * This allows libretro to query in absolute coordinates where on the * screen a mouse (or something similar) is being placed. * For a touch centric device, coordinates reported are the coordinates * of the press. @@ -148,33 +157,34 @@ extern "C" { * Coordinates in X and Y are reported as: * [-0x7fff, 0x7fff]: -0x7fff corresponds to the far left/top of the screen, * and 0x7fff corresponds to the far right/bottom of the screen. - * The "screen" is here defined as area that is passed to the frontend and + * The "screen" is here defined as area that is passed to the frontend and * later displayed on the monitor. * * The frontend is free to scale/resize this screen as it sees fit, however, - * (X, Y) = (-0x7fff, -0x7fff) will correspond to the top-left pixel of the + * (X, Y) = (-0x7fff, -0x7fff) will correspond to the top-left pixel of the * game image, etc. * - * To check if the pointer coordinates are valid (e.g. a touch display + * To check if the pointer coordinates are valid (e.g. a touch display * actually being touched), PRESSED returns 1 or 0. * - * If using a mouse on a desktop, PRESSED will usually correspond to the + * If using a mouse on a desktop, PRESSED will usually correspond to the * left mouse button, but this is a frontend decision. * PRESSED will only return 1 if the pointer is inside the game screen. * - * For multi-touch, the index variable can be used to successively query + * For multi-touch, the index variable can be used to successively query * more presses. * If index = 0 returns true for _PRESSED, coordinates can be extracted - * with _X, _Y for index = 0. One can then query _PRESSED, _X, _Y with + * with _X, _Y for index = 0. One can then query _PRESSED, _X, _Y with * index = 1, and so on. - * Eventually _PRESSED will return false for an index. No further presses + * Eventually _PRESSED will return false for an index. No further presses * are registered at this point. */ #define RETRO_DEVICE_POINTER 6 /* Buttons for the RetroPad (JOYPAD). - * The placement of these is equivalent to placements on the + * The placement of these is equivalent to placements on the * Super Nintendo controller. - * L2/R2/L3/R3 buttons correspond to the PS1 DualShock. */ + * L2/R2/L3/R3 buttons correspond to the PS1 DualShock. + * Also used as id values for RETRO_DEVICE_INDEX_ANALOG_BUTTON */ #define RETRO_DEVICE_ID_JOYPAD_B 0 #define RETRO_DEVICE_ID_JOYPAD_Y 1 #define RETRO_DEVICE_ID_JOYPAD_SELECT 2 @@ -192,11 +202,14 @@ extern "C" { #define RETRO_DEVICE_ID_JOYPAD_L3 14 #define RETRO_DEVICE_ID_JOYPAD_R3 15 +#define RETRO_DEVICE_ID_JOYPAD_MASK 256 + /* Index / Id values for ANALOG device. */ -#define RETRO_DEVICE_INDEX_ANALOG_LEFT 0 -#define RETRO_DEVICE_INDEX_ANALOG_RIGHT 1 -#define RETRO_DEVICE_ID_ANALOG_X 0 -#define RETRO_DEVICE_ID_ANALOG_Y 1 +#define RETRO_DEVICE_INDEX_ANALOG_LEFT 0 +#define RETRO_DEVICE_INDEX_ANALOG_RIGHT 1 +#define RETRO_DEVICE_INDEX_ANALOG_BUTTON 2 +#define RETRO_DEVICE_ID_ANALOG_X 0 +#define RETRO_DEVICE_ID_ANALOG_Y 1 /* Id values for MOUSE. */ #define RETRO_DEVICE_ID_MOUSE_X 0 @@ -208,20 +221,36 @@ extern "C" { #define RETRO_DEVICE_ID_MOUSE_MIDDLE 6 #define RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELUP 7 #define RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELDOWN 8 +#define RETRO_DEVICE_ID_MOUSE_BUTTON_4 9 +#define RETRO_DEVICE_ID_MOUSE_BUTTON_5 10 -/* Id values for LIGHTGUN types. */ -#define RETRO_DEVICE_ID_LIGHTGUN_X 0 -#define RETRO_DEVICE_ID_LIGHTGUN_Y 1 -#define RETRO_DEVICE_ID_LIGHTGUN_TRIGGER 2 -#define RETRO_DEVICE_ID_LIGHTGUN_CURSOR 3 -#define RETRO_DEVICE_ID_LIGHTGUN_TURBO 4 -#define RETRO_DEVICE_ID_LIGHTGUN_PAUSE 5 -#define RETRO_DEVICE_ID_LIGHTGUN_START 6 +/* Id values for LIGHTGUN. */ +#define RETRO_DEVICE_ID_LIGHTGUN_SCREEN_X 13 /*Absolute Position*/ +#define RETRO_DEVICE_ID_LIGHTGUN_SCREEN_Y 14 /*Absolute*/ +#define RETRO_DEVICE_ID_LIGHTGUN_IS_OFFSCREEN 15 /*Status Check*/ +#define RETRO_DEVICE_ID_LIGHTGUN_TRIGGER 2 +#define RETRO_DEVICE_ID_LIGHTGUN_RELOAD 16 /*Forced off-screen shot*/ +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_A 3 +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_B 4 +#define RETRO_DEVICE_ID_LIGHTGUN_START 6 +#define RETRO_DEVICE_ID_LIGHTGUN_SELECT 7 +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_C 8 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_UP 9 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_DOWN 10 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_LEFT 11 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_RIGHT 12 +/* deprecated */ +#define RETRO_DEVICE_ID_LIGHTGUN_X 0 /*Relative Position*/ +#define RETRO_DEVICE_ID_LIGHTGUN_Y 1 /*Relative*/ +#define RETRO_DEVICE_ID_LIGHTGUN_CURSOR 3 /*Use Aux:A*/ +#define RETRO_DEVICE_ID_LIGHTGUN_TURBO 4 /*Use Aux:B*/ +#define RETRO_DEVICE_ID_LIGHTGUN_PAUSE 5 /*Use Start*/ /* Id values for POINTER. */ #define RETRO_DEVICE_ID_POINTER_X 0 #define RETRO_DEVICE_ID_POINTER_Y 1 #define RETRO_DEVICE_ID_POINTER_PRESSED 2 +#define RETRO_DEVICE_ID_POINTER_COUNT 3 /* Returned from retro_get_region(). */ #define RETRO_REGION_NTSC 0 @@ -230,28 +259,33 @@ extern "C" { /* Id values for LANGUAGE */ enum retro_language { - RETRO_LANGUAGE_ENGLISH = 0, - RETRO_LANGUAGE_JAPANESE = 1, - RETRO_LANGUAGE_FRENCH = 2, - RETRO_LANGUAGE_SPANISH = 3, - RETRO_LANGUAGE_GERMAN = 4, - RETRO_LANGUAGE_ITALIAN = 5, - RETRO_LANGUAGE_DUTCH = 6, - RETRO_LANGUAGE_PORTUGUESE = 7, - RETRO_LANGUAGE_RUSSIAN = 8, - RETRO_LANGUAGE_KOREAN = 9, - RETRO_LANGUAGE_CHINESE_TRADITIONAL = 10, - RETRO_LANGUAGE_CHINESE_SIMPLIFIED = 11, - RETRO_LANGUAGE_ESPERANTO = 12, - RETRO_LANGUAGE_POLISH = 13, + RETRO_LANGUAGE_ENGLISH = 0, + RETRO_LANGUAGE_JAPANESE = 1, + RETRO_LANGUAGE_FRENCH = 2, + RETRO_LANGUAGE_SPANISH = 3, + RETRO_LANGUAGE_GERMAN = 4, + RETRO_LANGUAGE_ITALIAN = 5, + RETRO_LANGUAGE_DUTCH = 6, + RETRO_LANGUAGE_PORTUGUESE_BRAZIL = 7, + RETRO_LANGUAGE_PORTUGUESE_PORTUGAL = 8, + RETRO_LANGUAGE_RUSSIAN = 9, + RETRO_LANGUAGE_KOREAN = 10, + RETRO_LANGUAGE_CHINESE_TRADITIONAL = 11, + RETRO_LANGUAGE_CHINESE_SIMPLIFIED = 12, + RETRO_LANGUAGE_ESPERANTO = 13, + RETRO_LANGUAGE_POLISH = 14, + RETRO_LANGUAGE_VIETNAMESE = 15, + RETRO_LANGUAGE_ARABIC = 16, + RETRO_LANGUAGE_GREEK = 17, + RETRO_LANGUAGE_TURKISH = 18, RETRO_LANGUAGE_LAST, /* Ensure sizeof(enum) == sizeof(int) */ - RETRO_LANGUAGE_DUMMY = INT_MAX + RETRO_LANGUAGE_DUMMY = INT_MAX }; /* Passed to retro_get_memory_data/size(). - * If the memory type doesn't apply to the + * If the memory type doesn't apply to the * implementation NULL/0 can be returned. */ #define RETRO_MEMORY_MASK 0xff @@ -349,6 +383,10 @@ enum retro_key RETROK_x = 120, RETROK_y = 121, RETROK_z = 122, + RETROK_LEFTBRACE = 123, + RETROK_BAR = 124, + RETROK_RIGHTBRACE = 125, + RETROK_TILDE = 126, RETROK_DELETE = 127, RETROK_KP0 = 256, @@ -419,6 +457,7 @@ enum retro_key RETROK_POWER = 320, RETROK_EURO = 321, RETROK_UNDO = 322, + RETROK_OEM_102 = 323, RETROK_LAST, @@ -441,7 +480,7 @@ enum retro_mod RETROKMOD_DUMMY = INT_MAX /* Ensure sizeof(enum) == sizeof(int) */ }; -/* If set, this call is not part of the public libretro API yet. It can +/* If set, this call is not part of the public libretro API yet. It can * change or be removed at any time. */ #define RETRO_ENVIRONMENT_EXPERIMENTAL 0x10000 /* Environment callback to be used internally in frontend. */ @@ -450,12 +489,14 @@ enum retro_mod /* Environment commands. */ #define RETRO_ENVIRONMENT_SET_ROTATION 1 /* const unsigned * -- * Sets screen rotation of graphics. - * Is only implemented if rotation can be accelerated by hardware. - * Valid values are 0, 1, 2, 3, which rotates screen by 0, 90, 180, + * Valid values are 0, 1, 2, 3, which rotates screen by 0, 90, 180, * 270 degrees counter-clockwise respectively. */ #define RETRO_ENVIRONMENT_GET_OVERSCAN 2 /* bool * -- - * Boolean value whether or not the implementation should use overscan, + * NOTE: As of 2019 this callback is considered deprecated in favor of + * using core options to manage overscan in a more nuanced, core-specific way. + * + * Boolean value whether or not the implementation should use overscan, * or crop away overscan. */ #define RETRO_ENVIRONMENT_GET_CAN_DUPE 3 /* bool * -- @@ -463,15 +504,15 @@ enum retro_mod * passing NULL to video frame callback. */ - /* Environ 4, 5 are no longer supported (GET_VARIABLE / SET_VARIABLES), + /* Environ 4, 5 are no longer supported (GET_VARIABLE / SET_VARIABLES), * and reserved to avoid possible ABI clash. */ #define RETRO_ENVIRONMENT_SET_MESSAGE 6 /* const struct retro_message * -- - * Sets a message to be displayed in implementation-specific manner + * Sets a message to be displayed in implementation-specific manner * for a certain amount of 'frames'. - * Should not be used for trivial messages, which should simply be - * logged via RETRO_ENVIRONMENT_GET_LOG_INTERFACE (or as a + * Should not be used for trivial messages, which should simply be + * logged via RETRO_ENVIRONMENT_GET_LOG_INTERFACE (or as a * fallback, stderr). */ #define RETRO_ENVIRONMENT_SHUTDOWN 7 /* N/A (NULL) -- @@ -499,15 +540,15 @@ enum retro_mod #define RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY 9 /* const char ** -- * Returns the "system" directory of the frontend. - * This directory can be used to store system specific + * This directory can be used to store system specific * content such as BIOSes, configuration data, etc. * The returned value can be NULL. * If so, no such directory is defined, * and it's up to the implementation to find a suitable directory. * - * NOTE: Some cores used this folder also for "save" data such as + * NOTE: Some cores used this folder also for "save" data such as * memory cards, etc, for lack of a better place to put it. - * This is now discouraged, and if possible, cores should try to + * This is now discouraged, and if possible, cores should try to * use the new GET_SAVE_DIRECTORY. */ #define RETRO_ENVIRONMENT_SET_PIXEL_FORMAT 10 @@ -515,19 +556,19 @@ enum retro_mod * Sets the internal pixel format used by the implementation. * The default pixel format is RETRO_PIXEL_FORMAT_0RGB1555. * This pixel format however, is deprecated (see enum retro_pixel_format). - * If the call returns false, the frontend does not support this pixel + * If the call returns false, the frontend does not support this pixel * format. * - * This function should be called inside retro_load_game() or + * This function should be called inside retro_load_game() or * retro_get_system_av_info(). */ #define RETRO_ENVIRONMENT_SET_INPUT_DESCRIPTORS 11 /* const struct retro_input_descriptor * -- * Sets an array of retro_input_descriptors. * It is up to the frontend to present this in a usable way. - * The array is terminated by retro_input_descriptor::description + * The array is terminated by retro_input_descriptor::description * being set to NULL. - * This function can be called at any time, but it is recommended + * This function can be called at any time, but it is recommended * to call it as early as possible. */ #define RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK 12 @@ -536,52 +577,55 @@ enum retro_mod */ #define RETRO_ENVIRONMENT_SET_DISK_CONTROL_INTERFACE 13 /* const struct retro_disk_control_callback * -- - * Sets an interface which frontend can use to eject and insert + * Sets an interface which frontend can use to eject and insert * disk images. - * This is used for games which consist of multiple images and + * This is used for games which consist of multiple images and * must be manually swapped out by the user (e.g. PSX). */ #define RETRO_ENVIRONMENT_SET_HW_RENDER 14 /* struct retro_hw_render_callback * -- - * Sets an interface to let a libretro core render with + * Sets an interface to let a libretro core render with * hardware acceleration. * Should be called in retro_load_game(). - * If successful, libretro cores will be able to render to a + * If successful, libretro cores will be able to render to a * frontend-provided framebuffer. - * The size of this framebuffer will be at least as large as + * The size of this framebuffer will be at least as large as * max_width/max_height provided in get_av_info(). - * If HW rendering is used, pass only RETRO_HW_FRAME_BUFFER_VALID or + * If HW rendering is used, pass only RETRO_HW_FRAME_BUFFER_VALID or * NULL to retro_video_refresh_t. */ #define RETRO_ENVIRONMENT_GET_VARIABLE 15 /* struct retro_variable * -- * Interface to acquire user-defined information from environment * that cannot feasibly be supported in a multi-system way. - * 'key' should be set to a key which has already been set by + * 'key' should be set to a key which has already been set by * SET_VARIABLES. * 'data' will be set to a value or NULL. */ #define RETRO_ENVIRONMENT_SET_VARIABLES 16 /* const struct retro_variable * -- * Allows an implementation to signal the environment - * which variables it might want to check for later using + * which variables it might want to check for later using * GET_VARIABLE. - * This allows the frontend to present these variables to + * This allows the frontend to present these variables to * a user dynamically. - * This should be called as early as possible (ideally in - * retro_set_environment). + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterward it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. * - * 'data' points to an array of retro_variable structs + * 'data' points to an array of retro_variable structs * terminated by a { NULL, NULL } element. - * retro_variable::key should be namespaced to not collide - * with other implementations' keys. E.g. A core called + * retro_variable::key should be namespaced to not collide + * with other implementations' keys. E.g. A core called * 'foo' should use keys named as 'foo_option'. - * retro_variable::value should contain a human readable - * description of the key as well as a '|' delimited list + * retro_variable::value should contain a human readable + * description of the key as well as a '|' delimited list * of expected values. * - * The number of possible options should be very limited, - * i.e. it should be feasible to cycle through options + * The number of possible options should be very limited, + * i.e. it should be feasible to cycle through options * without a keyboard. * * First entry should be treated as a default. @@ -589,11 +633,11 @@ enum retro_mod * Example entry: * { "foo_option", "Speed hack coprocessor X; false|true" } * - * Text before first ';' is description. This ';' must be - * followed by a space, and followed by a list of possible + * Text before first ';' is description. This ';' must be + * followed by a space, and followed by a list of possible * values split up with '|'. * - * Only strings are operated on. The possible values will + * Only strings are operated on. The possible values will * generally be displayed and stored as-is by the frontend. */ #define RETRO_ENVIRONMENT_GET_VARIABLE_UPDATE 17 @@ -604,72 +648,72 @@ enum retro_mod */ #define RETRO_ENVIRONMENT_SET_SUPPORT_NO_GAME 18 /* const bool * -- - * If true, the libretro implementation supports calls to + * If true, the libretro implementation supports calls to * retro_load_game() with NULL as argument. * Used by cores which can run without particular game data. * This should be called within retro_set_environment() only. */ #define RETRO_ENVIRONMENT_GET_LIBRETRO_PATH 19 /* const char ** -- - * Retrieves the absolute path from where this libretro + * Retrieves the absolute path from where this libretro * implementation was loaded. - * NULL is returned if the libretro was loaded statically - * (i.e. linked statically to frontend), or if the path cannot be + * NULL is returned if the libretro was loaded statically + * (i.e. linked statically to frontend), or if the path cannot be * determined. - * Mostly useful in cooperation with SET_SUPPORT_NO_GAME as assets can + * Mostly useful in cooperation with SET_SUPPORT_NO_GAME as assets can * be loaded without ugly hacks. */ - - /* Environment 20 was an obsolete version of SET_AUDIO_CALLBACK. + + /* Environment 20 was an obsolete version of SET_AUDIO_CALLBACK. * It was not used by any known core at the time, * and was removed from the API. */ +#define RETRO_ENVIRONMENT_SET_FRAME_TIME_CALLBACK 21 + /* const struct retro_frame_time_callback * -- + * Lets the core know how much time has passed since last + * invocation of retro_run(). + * The frontend can tamper with the timing to fake fast-forward, + * slow-motion, frame stepping, etc. + * In this case the delta time will use the reference value + * in frame_time_callback.. + */ #define RETRO_ENVIRONMENT_SET_AUDIO_CALLBACK 22 /* const struct retro_audio_callback * -- - * Sets an interface which is used to notify a libretro core about audio + * Sets an interface which is used to notify a libretro core about audio * being available for writing. - * The callback can be called from any thread, so a core using this must + * The callback can be called from any thread, so a core using this must * have a thread safe audio implementation. - * It is intended for games where audio and video are completely + * It is intended for games where audio and video are completely * asynchronous and audio can be generated on the fly. - * This interface is not recommended for use with emulators which have + * This interface is not recommended for use with emulators which have * highly synchronous audio. * - * The callback only notifies about writability; the libretro core still + * The callback only notifies about writability; the libretro core still * has to call the normal audio callbacks - * to write audio. The audio callbacks must be called from within the + * to write audio. The audio callbacks must be called from within the * notification callback. * The amount of audio data to write is up to the implementation. * Generally, the audio callback will be called continously in a loop. * - * Due to thread safety guarantees and lack of sync between audio and - * video, a frontend can selectively disallow this interface based on - * internal configuration. A core using this interface must also + * Due to thread safety guarantees and lack of sync between audio and + * video, a frontend can selectively disallow this interface based on + * internal configuration. A core using this interface must also * implement the "normal" audio interface. * - * A libretro core using SET_AUDIO_CALLBACK should also make use of + * A libretro core using SET_AUDIO_CALLBACK should also make use of * SET_FRAME_TIME_CALLBACK. */ -#define RETRO_ENVIRONMENT_SET_FRAME_TIME_CALLBACK 21 - /* const struct retro_frame_time_callback * -- - * Lets the core know how much time has passed since last - * invocation of retro_run(). - * The frontend can tamper with the timing to fake fast-forward, - * slow-motion, frame stepping, etc. - * In this case the delta time will use the reference value - * in frame_time_callback.. - */ #define RETRO_ENVIRONMENT_GET_RUMBLE_INTERFACE 23 /* struct retro_rumble_interface * -- - * Gets an interface which is used by a libretro core to set + * Gets an interface which is used by a libretro core to set * state of rumble motors in controllers. - * A strong and weak motor is supported, and they can be + * A strong and weak motor is supported, and they can be * controlled indepedently. */ #define RETRO_ENVIRONMENT_GET_INPUT_DEVICE_CAPABILITIES 24 /* uint64_t * -- - * Gets a bitmask telling which device type are expected to be + * Gets a bitmask telling which device type are expected to be * handled properly in a call to retro_input_state_t. - * Devices which are not handled or recognized always return + * Devices which are not handled or recognized always return * 0 in retro_input_state_t. * Example bitmask: caps = (1 << RETRO_DEVICE_JOYPAD) | (1 << RETRO_DEVICE_ANALOG). * Should only be called in retro_run(). @@ -678,56 +722,56 @@ enum retro_mod /* struct retro_sensor_interface * -- * Gets access to the sensor interface. * The purpose of this interface is to allow - * setting state related to sensors such as polling rate, + * setting state related to sensors such as polling rate, * enabling/disable it entirely, etc. - * Reading sensor state is done via the normal + * Reading sensor state is done via the normal * input_state_callback API. */ #define RETRO_ENVIRONMENT_GET_CAMERA_INTERFACE (26 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* struct retro_camera_callback * -- * Gets an interface to a video camera driver. - * A libretro core can use this interface to get access to a + * A libretro core can use this interface to get access to a * video camera. - * New video frames are delivered in a callback in same + * New video frames are delivered in a callback in same * thread as retro_run(). * * GET_CAMERA_INTERFACE should be called in retro_load_game(). * - * Depending on the camera implementation used, camera frames + * Depending on the camera implementation used, camera frames * will be delivered as a raw framebuffer, * or as an OpenGL texture directly. * - * The core has to tell the frontend here which types of + * The core has to tell the frontend here which types of * buffers can be handled properly. - * An OpenGL texture can only be handled when using a + * An OpenGL texture can only be handled when using a * libretro GL core (SET_HW_RENDER). - * It is recommended to use a libretro GL core when + * It is recommended to use a libretro GL core when * using camera interface. * - * The camera is not started automatically. The retrieved start/stop + * The camera is not started automatically. The retrieved start/stop * functions must be used to explicitly * start and stop the camera driver. */ #define RETRO_ENVIRONMENT_GET_LOG_INTERFACE 27 /* struct retro_log_callback * -- - * Gets an interface for logging. This is useful for + * Gets an interface for logging. This is useful for * logging in a cross-platform way - * as certain platforms cannot use use stderr for logging. + * as certain platforms cannot use stderr for logging. * It also allows the frontend to * show logging information in a more suitable way. - * If this interface is not used, libretro cores should + * If this interface is not used, libretro cores should * log to stderr as desired. */ #define RETRO_ENVIRONMENT_GET_PERF_INTERFACE 28 /* struct retro_perf_callback * -- - * Gets an interface for performance counters. This is useful - * for performance logging in a cross-platform way and for detecting + * Gets an interface for performance counters. This is useful + * for performance logging in a cross-platform way and for detecting * architecture-specific features, such as SIMD support. */ #define RETRO_ENVIRONMENT_GET_LOCATION_INTERFACE 29 /* struct retro_location_callback * -- * Gets access to the location interface. - * The purpose of this interface is to be able to retrieve + * The purpose of this interface is to be able to retrieve * location-based information from the host device, * such as current latitude / longitude. */ @@ -735,7 +779,7 @@ enum retro_mod #define RETRO_ENVIRONMENT_GET_CORE_ASSETS_DIRECTORY 30 /* const char ** -- * Returns the "core assets" directory of the frontend. - * This directory can be used to store specific assets that the + * This directory can be used to store specific assets that the * core relies upon, such as art assets, * input data, etc etc. * The returned value can be NULL. @@ -744,76 +788,77 @@ enum retro_mod */ #define RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY 31 /* const char ** -- - * Returns the "save" directory of the frontend. - * This directory can be used to store SRAM, memory cards, - * high scores, etc, if the libretro core + * Returns the "save" directory of the frontend, unless there is no + * save directory available. The save directory should be used to + * store SRAM, memory cards, high scores, etc, if the libretro core * cannot use the regular memory interface (retro_get_memory_data()). * - * NOTE: libretro cores used to check GET_SYSTEM_DIRECTORY for - * similar things before. - * They should still check GET_SYSTEM_DIRECTORY if they want to - * be backwards compatible. - * The path here can be NULL. It should only be non-NULL if the - * frontend user has set a specific save path. + * If the frontend cannot designate a save directory, it will return + * NULL to indicate that the core should attempt to operate without a + * save directory set. + * + * NOTE: early libretro cores used the system directory for save + * files. Cores that need to be backwards-compatible can still check + * GET_SYSTEM_DIRECTORY. */ #define RETRO_ENVIRONMENT_SET_SYSTEM_AV_INFO 32 /* const struct retro_system_av_info * -- - * Sets a new av_info structure. This can only be called from + * Sets a new av_info structure. This can only be called from * within retro_run(). - * This should *only* be used if the core is completely altering the + * This should *only* be used if the core is completely altering the * internal resolutions, aspect ratios, timings, sampling rate, etc. - * Calling this can require a full reinitialization of video/audio + * Calling this can require a full reinitialization of video/audio * drivers in the frontend, * - * so it is important to call it very sparingly, and usually only with + * so it is important to call it very sparingly, and usually only with * the users explicit consent. - * An eventual driver reinitialize will happen so that video and + * An eventual driver reinitialize will happen so that video and * audio callbacks - * happening after this call within the same retro_run() call will + * happening after this call within the same retro_run() call will * target the newly initialized driver. * - * This callback makes it possible to support configurable resolutions + * This callback makes it possible to support configurable resolutions * in games, which can be useful to * avoid setting the "worst case" in max_width/max_height. * - * ***HIGHLY RECOMMENDED*** Do not call this callback every time + * ***HIGHLY RECOMMENDED*** Do not call this callback every time * resolution changes in an emulator core if it's - * expected to be a temporary change, for the reasons of possible + * expected to be a temporary change, for the reasons of possible * driver reinitialization. - * This call is not a free pass for not trying to provide - * correct values in retro_get_system_av_info(). If you need to change - * things like aspect ratio or nominal width/height, - * use RETRO_ENVIRONMENT_SET_GEOMETRY, which is a softer variant + * This call is not a free pass for not trying to provide + * correct values in retro_get_system_av_info(). If you need to change + * things like aspect ratio or nominal width/height, + * use RETRO_ENVIRONMENT_SET_GEOMETRY, which is a softer variant * of SET_SYSTEM_AV_INFO. * - * If this returns false, the frontend does not acknowledge a + * If this returns false, the frontend does not acknowledge a * changed av_info struct. */ #define RETRO_ENVIRONMENT_SET_PROC_ADDRESS_CALLBACK 33 /* const struct retro_get_proc_address_interface * -- - * Allows a libretro core to announce support for the + * Allows a libretro core to announce support for the * get_proc_address() interface. - * This interface allows for a standard way to extend libretro where + * This interface allows for a standard way to extend libretro where * use of environment calls are too indirect, * e.g. for cases where the frontend wants to call directly into the core. * - * If a core wants to expose this interface, SET_PROC_ADDRESS_CALLBACK + * If a core wants to expose this interface, SET_PROC_ADDRESS_CALLBACK * **MUST** be called from within retro_set_environment(). */ #define RETRO_ENVIRONMENT_SET_SUBSYSTEM_INFO 34 /* const struct retro_subsystem_info * -- * This environment call introduces the concept of libretro "subsystems". - * A subsystem is a variant of a libretro core which supports + * A subsystem is a variant of a libretro core which supports * different kinds of games. - * The purpose of this is to support e.g. emulators which might + * The purpose of this is to support e.g. emulators which might * have special needs, e.g. Super Nintendo's Super GameBoy, Sufami Turbo. - * It can also be used to pick among subsystems in an explicit way + * It can also be used to pick among subsystems in an explicit way * if the libretro implementation is a multi-system emulator itself. * * Loading a game via a subsystem is done with retro_load_game_special(), - * and this environment call allows a libretro core to expose which + * and this environment call allows a libretro core to expose which * subsystems are supported for use with retro_load_game_special(). - * A core passes an array of retro_game_special_info which is terminated + * A core passes an array of retro_game_special_info which is terminated * with a zeroed out retro_game_special_info struct. * * If a core wants to use this functionality, SET_SUBSYSTEM_INFO @@ -821,68 +866,81 @@ enum retro_mod */ #define RETRO_ENVIRONMENT_SET_CONTROLLER_INFO 35 /* const struct retro_controller_info * -- - * This environment call lets a libretro core tell the frontend - * which controller types are recognized in calls to + * This environment call lets a libretro core tell the frontend + * which controller subclasses are recognized in calls to * retro_set_controller_port_device(). * - * Some emulators such as Super Nintendo - * support multiple lightgun types which must be specifically - * selected from. - * It is therefore sometimes necessary for a frontend to be able - * to tell the core about a special kind of input device which is - * not covered by the libretro input API. + * Some emulators such as Super Nintendo support multiple lightgun + * types which must be specifically selected from. It is therefore + * sometimes necessary for a frontend to be able to tell the core + * about a special kind of input device which is not specifcally + * provided by the Libretro API. * - * In order for a frontend to understand the workings of an input device, - * it must be a specialized type - * of the generic device types already defined in the libretro API. + * In order for a frontend to understand the workings of those devices, + * they must be defined as a specialized subclass of the generic device + * types already defined in the libretro API. * - * Which devices are supported can vary per input port. - * The core must pass an array of const struct retro_controller_info which - * is terminated with a blanked out struct. Each element of the struct - * corresponds to an ascending port index to - * retro_set_controller_port_device(). - * Even if special device types are set in the libretro core, + * The core must pass an array of const struct retro_controller_info which + * is terminated with a blanked out struct. Each element of the + * retro_controller_info struct corresponds to the ascending port index + * that is passed to retro_set_controller_port_device() when that function + * is called to indicate to the core that the frontend has changed the + * active device subclass. SEE ALSO: retro_set_controller_port_device() + * + * The ascending input port indexes provided by the core in the struct + * are generally presented by frontends as ascending User # or Player #, + * such as Player 1, Player 2, Player 3, etc. Which device subclasses are + * supported can vary per input port. + * + * The first inner element of each entry in the retro_controller_info array + * is a retro_controller_description struct that specifies the names and + * codes of all device subclasses that are available for the corresponding + * User or Player, beginning with the generic Libretro device that the + * subclasses are derived from. The second inner element of each entry is the + * total number of subclasses that are listed in the retro_controller_description. + * + * NOTE: Even if special device types are set in the libretro core, * libretro should only poll input based on the base input device types. */ #define RETRO_ENVIRONMENT_SET_MEMORY_MAPS (36 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const struct retro_memory_map * -- - * This environment call lets a libretro core tell the frontend + * This environment call lets a libretro core tell the frontend * about the memory maps this core emulates. * This can be used to implement, for example, cheats in a core-agnostic way. * - * Should only be used by emulators; it doesn't make much sense for + * Should only be used by emulators; it doesn't make much sense for * anything else. - * It is recommended to expose all relevant pointers through + * It is recommended to expose all relevant pointers through * retro_get_memory_* as well. * * Can be called from retro_init and retro_load_game. */ #define RETRO_ENVIRONMENT_SET_GEOMETRY 37 /* const struct retro_game_geometry * -- - * This environment call is similar to SET_SYSTEM_AV_INFO for changing - * video parameters, but provides a guarantee that drivers will not be + * This environment call is similar to SET_SYSTEM_AV_INFO for changing + * video parameters, but provides a guarantee that drivers will not be * reinitialized. * This can only be called from within retro_run(). * - * The purpose of this call is to allow a core to alter nominal - * width/heights as well as aspect ratios on-the-fly, which can be + * The purpose of this call is to allow a core to alter nominal + * width/heights as well as aspect ratios on-the-fly, which can be * useful for some emulators to change in run-time. * * max_width/max_height arguments are ignored and cannot be changed - * with this call as this could potentially require a reinitialization or a + * with this call as this could potentially require a reinitialization or a * non-constant time operation. * If max_width/max_height are to be changed, SET_SYSTEM_AV_INFO is required. * - * A frontend must guarantee that this environment call completes in + * A frontend must guarantee that this environment call completes in * constant time. */ -#define RETRO_ENVIRONMENT_GET_USERNAME 38 +#define RETRO_ENVIRONMENT_GET_USERNAME 38 /* const char ** * Returns the specified username of the frontend, if specified by the user. - * This username can be used as a nickname for a core that has online facilities + * This username can be used as a nickname for a core that has online facilities * or any other mode where personalization of the user is desirable. * The returned value can be NULL. - * If this environ callback is used by a core that requires a valid username, + * If this environ callback is used by a core that requires a valid username, * a default username should be specified by the core. */ #define RETRO_ENVIRONMENT_GET_LANGUAGE 39 @@ -920,20 +978,6 @@ enum retro_mod * A frontend must make sure that the pointer obtained from this function is * writeable (and readable). */ - -enum retro_hw_render_interface_type -{ - RETRO_HW_RENDER_INTERFACE_VULKAN = 0, - RETRO_HW_RENDER_INTERFACE_DUMMY = INT_MAX -}; - -/* Base struct. All retro_hw_render_interface_* types - * contain at least these fields. */ -struct retro_hw_render_interface -{ - enum retro_hw_render_interface_type interface_type; - unsigned interface_version; -}; #define RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE (41 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const struct retro_hw_render_interface ** -- * Returns an API specific rendering interface for accessing API specific data. @@ -945,7 +989,6 @@ struct retro_hw_render_interface * Similarly, after context_destroyed callback returns, * the contents of the HW_RENDER_INTERFACE are invalidated. */ - #define RETRO_ENVIRONMENT_SET_SUPPORT_ACHIEVEMENTS (42 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const bool * -- * If true, the libretro implementation supports achievements @@ -954,6 +997,481 @@ struct retro_hw_render_interface * * This must be called before the first call to retro_run. */ +#define RETRO_ENVIRONMENT_SET_HW_RENDER_CONTEXT_NEGOTIATION_INTERFACE (43 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* const struct retro_hw_render_context_negotiation_interface * -- + * Sets an interface which lets the libretro core negotiate with frontend how a context is created. + * The semantics of this interface depends on which API is used in SET_HW_RENDER earlier. + * This interface will be used when the frontend is trying to create a HW rendering context, + * so it will be used after SET_HW_RENDER, but before the context_reset callback. + */ +#define RETRO_ENVIRONMENT_SET_SERIALIZATION_QUIRKS 44 + /* uint64_t * -- + * Sets quirk flags associated with serialization. The frontend will zero any flags it doesn't + * recognize or support. Should be set in either retro_init or retro_load_game, but not both. + */ +#define RETRO_ENVIRONMENT_SET_HW_SHARED_CONTEXT (44 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* N/A (null) * -- + * The frontend will try to use a 'shared' hardware context (mostly applicable + * to OpenGL) when a hardware context is being set up. + * + * Returns true if the frontend supports shared hardware contexts and false + * if the frontend does not support shared hardware contexts. + * + * This will do nothing on its own until SET_HW_RENDER env callbacks are + * being used. + */ +#define RETRO_ENVIRONMENT_GET_VFS_INTERFACE (45 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_vfs_interface_info * -- + * Gets access to the VFS interface. + * VFS presence needs to be queried prior to load_game or any + * get_system/save/other_directory being called to let front end know + * core supports VFS before it starts handing out paths. + * It is recomended to do so in retro_set_environment + */ +#define RETRO_ENVIRONMENT_GET_LED_INTERFACE (46 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_led_interface * -- + * Gets an interface which is used by a libretro core to set + * state of LEDs. + */ +#define RETRO_ENVIRONMENT_GET_AUDIO_VIDEO_ENABLE (47 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* int * -- + * Tells the core if the frontend wants audio or video. + * If disabled, the frontend will discard the audio or video, + * so the core may decide to skip generating a frame or generating audio. + * This is mainly used for increasing performance. + * Bit 0 (value 1): Enable Video + * Bit 1 (value 2): Enable Audio + * Bit 2 (value 4): Use Fast Savestates. + * Bit 3 (value 8): Hard Disable Audio + * Other bits are reserved for future use and will default to zero. + * If video is disabled: + * * The frontend wants the core to not generate any video, + * including presenting frames via hardware acceleration. + * * The frontend's video frame callback will do nothing. + * * After running the frame, the video output of the next frame should be + * no different than if video was enabled, and saving and loading state + * should have no issues. + * If audio is disabled: + * * The frontend wants the core to not generate any audio. + * * The frontend's audio callbacks will do nothing. + * * After running the frame, the audio output of the next frame should be + * no different than if audio was enabled, and saving and loading state + * should have no issues. + * Fast Savestates: + * * Guaranteed to be created by the same binary that will load them. + * * Will not be written to or read from the disk. + * * Suggest that the core assumes loading state will succeed. + * * Suggest that the core updates its memory buffers in-place if possible. + * * Suggest that the core skips clearing memory. + * * Suggest that the core skips resetting the system. + * * Suggest that the core may skip validation steps. + * Hard Disable Audio: + * * Used for a secondary core when running ahead. + * * Indicates that the frontend will never need audio from the core. + * * Suggests that the core may stop synthesizing audio, but this should not + * compromise emulation accuracy. + * * Audio output for the next frame does not matter, and the frontend will + * never need an accurate audio state in the future. + * * State will never be saved when using Hard Disable Audio. + */ +#define RETRO_ENVIRONMENT_GET_MIDI_INTERFACE (48 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_midi_interface ** -- + * Returns a MIDI interface that can be used for raw data I/O. + */ + +#define RETRO_ENVIRONMENT_GET_FASTFORWARDING (49 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* bool * -- + * Boolean value that indicates whether or not the frontend is in + * fastforwarding mode. + */ + +#define RETRO_ENVIRONMENT_GET_TARGET_REFRESH_RATE (50 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* float * -- + * Float value that lets us know what target refresh rate + * is curently in use by the frontend. + * + * The core can use the returned value to set an ideal + * refresh rate/framerate. + */ + +#define RETRO_ENVIRONMENT_GET_INPUT_BITMASKS (51 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* bool * -- + * Boolean value that indicates whether or not the frontend supports + * input bitmasks being returned by retro_input_state_t. The advantage + * of this is that retro_input_state_t has to be only called once to + * grab all button states instead of multiple times. + * + * If it returns true, you can pass RETRO_DEVICE_ID_JOYPAD_MASK as 'id' + * to retro_input_state_t (make sure 'device' is set to RETRO_DEVICE_JOYPAD). + * It will return a bitmask of all the digital buttons. + */ + +#define RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION 52 + /* unsigned * -- + * Unsigned value is the API version number of the core options + * interface supported by the frontend. If callback return false, + * API version is assumed to be 0. + * + * In legacy code, core options are set by passing an array of + * retro_variable structs to RETRO_ENVIRONMENT_SET_VARIABLES. + * This may be still be done regardless of the core options + * interface version. + * + * If version is >= 1 however, core options may instead be set by + * passing an array of retro_core_option_definition structs to + * RETRO_ENVIRONMENT_SET_CORE_OPTIONS, or a 2D array of + * retro_core_option_definition structs to RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL. + * This allows the core to additionally set option sublabel information + * and/or provide localisation support. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS 53 + /* const struct retro_core_option_definition ** -- + * Allows an implementation to signal the environment + * which variables it might want to check for later using + * GET_VARIABLE. + * This allows the frontend to present these variables to + * a user dynamically. + * This should only be called if RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION + * returns an API version of >= 1. + * This should be called instead of RETRO_ENVIRONMENT_SET_VARIABLES. + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterwards it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. + * + * 'data' points to an array of retro_core_option_definition structs + * terminated by a { NULL, NULL, NULL, {{0}}, NULL } element. + * retro_core_option_definition::key should be namespaced to not collide + * with other implementations' keys. e.g. A core called + * 'foo' should use keys named as 'foo_option'. + * retro_core_option_definition::desc should contain a human readable + * description of the key. + * retro_core_option_definition::info should contain any additional human + * readable information text that a typical user may need to + * understand the functionality of the option. + * retro_core_option_definition::values is an array of retro_core_option_value + * structs terminated by a { NULL, NULL } element. + * > retro_core_option_definition::values[index].value is an expected option + * value. + * > retro_core_option_definition::values[index].label is a human readable + * label used when displaying the value on screen. If NULL, + * the value itself is used. + * retro_core_option_definition::default_value is the default core option + * setting. It must match one of the expected option values in the + * retro_core_option_definition::values array. If it does not, or the + * default value is NULL, the first entry in the + * retro_core_option_definition::values array is treated as the default. + * + * The number of possible options should be very limited, + * and must be less than RETRO_NUM_CORE_OPTION_VALUES_MAX. + * i.e. it should be feasible to cycle through options + * without a keyboard. + * + * Example entry: + * { + * "foo_option", + * "Speed hack coprocessor X", + * "Provides increased performance at the expense of reduced accuracy", + * { + * { "false", NULL }, + * { "true", NULL }, + * { "unstable", "Turbo (Unstable)" }, + * { NULL, NULL }, + * }, + * "false" + * } + * + * Only strings are operated on. The possible values will + * generally be displayed and stored as-is by the frontend. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL 54 + /* const struct retro_core_options_intl * -- + * Allows an implementation to signal the environment + * which variables it might want to check for later using + * GET_VARIABLE. + * This allows the frontend to present these variables to + * a user dynamically. + * This should only be called if RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION + * returns an API version of >= 1. + * This should be called instead of RETRO_ENVIRONMENT_SET_VARIABLES. + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterwards it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. + * + * This is fundamentally the same as RETRO_ENVIRONMENT_SET_CORE_OPTIONS, + * with the addition of localisation support. The description of the + * RETRO_ENVIRONMENT_SET_CORE_OPTIONS callback should be consulted + * for further details. + * + * 'data' points to a retro_core_options_intl struct. + * + * retro_core_options_intl::us is a pointer to an array of + * retro_core_option_definition structs defining the US English + * core options implementation. It must point to a valid array. + * + * retro_core_options_intl::local is a pointer to an array of + * retro_core_option_definition structs defining core options for + * the current frontend language. It may be NULL (in which case + * retro_core_options_intl::us is used by the frontend). Any items + * missing from this array will be read from retro_core_options_intl::us + * instead. + * + * NOTE: Default core option values are always taken from the + * retro_core_options_intl::us array. Any default values in + * retro_core_options_intl::local array will be ignored. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY 55 + /* struct retro_core_option_display * -- + * + * Allows an implementation to signal the environment to show + * or hide a variable when displaying core options. This is + * considered a *suggestion*. The frontend is free to ignore + * this callback, and its implementation not considered mandatory. + * + * 'data' points to a retro_core_option_display struct + * + * retro_core_option_display::key is a variable identifier + * which has already been set by SET_VARIABLES/SET_CORE_OPTIONS. + * + * retro_core_option_display::visible is a boolean, specifying + * whether variable should be displayed + * + * Note that all core option variables will be set visible by + * default when calling SET_VARIABLES/SET_CORE_OPTIONS. + */ + +/* VFS functionality */ + +/* File paths: + * File paths passed as parameters when using this API shall be well formed UNIX-style, + * using "/" (unquoted forward slash) as directory separator regardless of the platform's native separator. + * Paths shall also include at least one forward slash ("game.bin" is an invalid path, use "./game.bin" instead). + * Other than the directory separator, cores shall not make assumptions about path format: + * "C:/path/game.bin", "http://example.com/game.bin", "#game/game.bin", "./game.bin" (without quotes) are all valid paths. + * Cores may replace the basename or remove path components from the end, and/or add new components; + * however, cores shall not append "./", "../" or multiple consecutive forward slashes ("//") to paths they request to front end. + * The frontend is encouraged to make such paths work as well as it can, but is allowed to give up if the core alters paths too much. + * Frontends are encouraged, but not required, to support native file system paths (modulo replacing the directory separator, if applicable). + * Cores are allowed to try using them, but must remain functional if the front rejects such requests. + * Cores are encouraged to use the libretro-common filestream functions for file I/O, + * as they seamlessly integrate with VFS, deal with directory separator replacement as appropriate + * and provide platform-specific fallbacks in cases where front ends do not support VFS. */ + +/* Opaque file handle + * Introduced in VFS API v1 */ +struct retro_vfs_file_handle; + +/* Opaque directory handle + * Introduced in VFS API v3 */ +struct retro_vfs_dir_handle; + +/* File open flags + * Introduced in VFS API v1 */ +#define RETRO_VFS_FILE_ACCESS_READ (1 << 0) /* Read only mode */ +#define RETRO_VFS_FILE_ACCESS_WRITE (1 << 1) /* Write only mode, discard contents and overwrites existing file unless RETRO_VFS_FILE_ACCESS_UPDATE is also specified */ +#define RETRO_VFS_FILE_ACCESS_READ_WRITE (RETRO_VFS_FILE_ACCESS_READ | RETRO_VFS_FILE_ACCESS_WRITE) /* Read-write mode, discard contents and overwrites existing file unless RETRO_VFS_FILE_ACCESS_UPDATE is also specified*/ +#define RETRO_VFS_FILE_ACCESS_UPDATE_EXISTING (1 << 2) /* Prevents discarding content of existing files opened for writing */ + +/* These are only hints. The frontend may choose to ignore them. Other than RAM/CPU/etc use, + and how they react to unlikely external interference (for example someone else writing to that file, + or the file's server going down), behavior will not change. */ +#define RETRO_VFS_FILE_ACCESS_HINT_NONE (0) +/* Indicate that the file will be accessed many times. The frontend should aggressively cache everything. */ +#define RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS (1 << 0) + +/* Seek positions */ +#define RETRO_VFS_SEEK_POSITION_START 0 +#define RETRO_VFS_SEEK_POSITION_CURRENT 1 +#define RETRO_VFS_SEEK_POSITION_END 2 + +/* stat() result flags + * Introduced in VFS API v3 */ +#define RETRO_VFS_STAT_IS_VALID (1 << 0) +#define RETRO_VFS_STAT_IS_DIRECTORY (1 << 1) +#define RETRO_VFS_STAT_IS_CHARACTER_SPECIAL (1 << 2) + +/* Get path from opaque handle. Returns the exact same path passed to file_open when getting the handle + * Introduced in VFS API v1 */ +typedef const char *(RETRO_CALLCONV *retro_vfs_get_path_t)(struct retro_vfs_file_handle *stream); + +/* Open a file for reading or writing. If path points to a directory, this will + * fail. Returns the opaque file handle, or NULL for error. + * Introduced in VFS API v1 */ +typedef struct retro_vfs_file_handle *(RETRO_CALLCONV *retro_vfs_open_t)(const char *path, unsigned mode, unsigned hints); + +/* Close the file and release its resources. Must be called if open_file returns non-NULL. Returns 0 on success, -1 on failure. + * Whether the call succeeds ot not, the handle passed as parameter becomes invalid and should no longer be used. + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_close_t)(struct retro_vfs_file_handle *stream); + +/* Return the size of the file in bytes, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_size_t)(struct retro_vfs_file_handle *stream); + +/* Truncate file to specified size. Returns 0 on success or -1 on error + * Introduced in VFS API v2 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_truncate_t)(struct retro_vfs_file_handle *stream, int64_t length); + +/* Get the current read / write position for the file. Returns -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_tell_t)(struct retro_vfs_file_handle *stream); + +/* Set the current read/write position for the file. Returns the new position, -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_seek_t)(struct retro_vfs_file_handle *stream, int64_t offset, int seek_position); + +/* Read data from a file. Returns the number of bytes read, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_read_t)(struct retro_vfs_file_handle *stream, void *s, uint64_t len); + +/* Write data to a file. Returns the number of bytes written, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_write_t)(struct retro_vfs_file_handle *stream, const void *s, uint64_t len); + +/* Flush pending writes to file, if using buffered IO. Returns 0 on sucess, or -1 on failure. + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_flush_t)(struct retro_vfs_file_handle *stream); + +/* Delete the specified file. Returns 0 on success, -1 on failure + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_remove_t)(const char *path); + +/* Rename the specified file. Returns 0 on success, -1 on failure + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_rename_t)(const char *old_path, const char *new_path); + +/* Stat the specified file. Retruns a bitmask of RETRO_VFS_STAT_* flags, none are set if path was not valid. + * Additionally stores file size in given variable, unless NULL is given. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_stat_t)(const char *path, int32_t *size); + +/* Create the specified directory. Returns 0 on success, -1 on unknown failure, -2 if already exists. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_mkdir_t)(const char *dir); + +/* Open the specified directory for listing. Returns the opaque dir handle, or NULL for error. + * Support for the include_hidden argument may vary depending on the platform. + * Introduced in VFS API v3 */ +typedef struct retro_vfs_dir_handle *(RETRO_CALLCONV *retro_vfs_opendir_t)(const char *dir, bool include_hidden); + +/* Read the directory entry at the current position, and move the read pointer to the next position. + * Returns true on success, false if already on the last entry. + * Introduced in VFS API v3 */ +typedef bool (RETRO_CALLCONV *retro_vfs_readdir_t)(struct retro_vfs_dir_handle *dirstream); + +/* Get the name of the last entry read. Returns a string on success, or NULL for error. + * The returned string pointer is valid until the next call to readdir or closedir. + * Introduced in VFS API v3 */ +typedef const char *(RETRO_CALLCONV *retro_vfs_dirent_get_name_t)(struct retro_vfs_dir_handle *dirstream); + +/* Check if the last entry read was a directory. Returns true if it was, false otherwise (or on error). + * Introduced in VFS API v3 */ +typedef bool (RETRO_CALLCONV *retro_vfs_dirent_is_dir_t)(struct retro_vfs_dir_handle *dirstream); + +/* Close the directory and release its resources. Must be called if opendir returns non-NULL. Returns 0 on success, -1 on failure. + * Whether the call succeeds ot not, the handle passed as parameter becomes invalid and should no longer be used. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_closedir_t)(struct retro_vfs_dir_handle *dirstream); + +struct retro_vfs_interface +{ + /* VFS API v1 */ + retro_vfs_get_path_t get_path; + retro_vfs_open_t open; + retro_vfs_close_t close; + retro_vfs_size_t size; + retro_vfs_tell_t tell; + retro_vfs_seek_t seek; + retro_vfs_read_t read; + retro_vfs_write_t write; + retro_vfs_flush_t flush; + retro_vfs_remove_t remove; + retro_vfs_rename_t rename; + /* VFS API v2 */ + retro_vfs_truncate_t truncate; + /* VFS API v3 */ + retro_vfs_stat_t stat; + retro_vfs_mkdir_t mkdir; + retro_vfs_opendir_t opendir; + retro_vfs_readdir_t readdir; + retro_vfs_dirent_get_name_t dirent_get_name; + retro_vfs_dirent_is_dir_t dirent_is_dir; + retro_vfs_closedir_t closedir; +}; + +struct retro_vfs_interface_info +{ + /* Set by core: should this be higher than the version the front end supports, + * front end will return false in the RETRO_ENVIRONMENT_GET_VFS_INTERFACE call + * Introduced in VFS API v1 */ + uint32_t required_interface_version; + + /* Frontend writes interface pointer here. The frontend also sets the actual + * version, must be at least required_interface_version. + * Introduced in VFS API v1 */ + struct retro_vfs_interface *iface; +}; + +enum retro_hw_render_interface_type +{ + RETRO_HW_RENDER_INTERFACE_VULKAN = 0, + RETRO_HW_RENDER_INTERFACE_D3D9 = 1, + RETRO_HW_RENDER_INTERFACE_D3D10 = 2, + RETRO_HW_RENDER_INTERFACE_D3D11 = 3, + RETRO_HW_RENDER_INTERFACE_D3D12 = 4, + RETRO_HW_RENDER_INTERFACE_GSKIT_PS2 = 5, + RETRO_HW_RENDER_INTERFACE_DUMMY = INT_MAX +}; + +/* Base struct. All retro_hw_render_interface_* types + * contain at least these fields. */ +struct retro_hw_render_interface +{ + enum retro_hw_render_interface_type interface_type; + unsigned interface_version; +}; + +typedef void (RETRO_CALLCONV *retro_set_led_state_t)(int led, int state); +struct retro_led_interface +{ + retro_set_led_state_t set_led_state; +}; + +/* Retrieves the current state of the MIDI input. + * Returns true if it's enabled, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_input_enabled_t)(void); + +/* Retrieves the current state of the MIDI output. + * Returns true if it's enabled, false otherwise */ +typedef bool (RETRO_CALLCONV *retro_midi_output_enabled_t)(void); + +/* Reads next byte from the input stream. + * Returns true if byte is read, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_read_t)(uint8_t *byte); + +/* Writes byte to the output stream. + * 'delta_time' is in microseconds and represent time elapsed since previous write. + * Returns true if byte is written, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_write_t)(uint8_t byte, uint32_t delta_time); + +/* Flushes previously written data. + * Returns true if successful, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_flush_t)(void); + +struct retro_midi_interface +{ + retro_midi_input_enabled_t input_enabled; + retro_midi_output_enabled_t output_enabled; + retro_midi_read_t read; + retro_midi_write_t write; + retro_midi_flush_t flush; +}; enum retro_hw_render_context_negotiation_interface_type { @@ -968,77 +1486,97 @@ struct retro_hw_render_context_negotiation_interface enum retro_hw_render_context_negotiation_interface_type interface_type; unsigned interface_version; }; -#define RETRO_ENVIRONMENT_SET_HW_RENDER_CONTEXT_NEGOTIATION_INTERFACE (43 | RETRO_ENVIRONMENT_EXPERIMENTAL) - /* const struct retro_hw_render_context_negotiation_interface * -- - * Sets an interface which lets the libretro core negotiate with frontend how a context is created. - * The semantics of this interface depends on which API is used in SET_HW_RENDER earlier. - * This interface will be used when the frontend is trying to create a HW rendering context, - * so it will be used after SET_HW_RENDER, but before the context_reset callback. - */ -#define RETRO_MEMDESC_CONST (1 << 0) /* The frontend will never change this memory area once retro_load_game has returned. */ -#define RETRO_MEMDESC_BIGENDIAN (1 << 1) /* The memory area contains big endian data. Default is little endian. */ -#define RETRO_MEMDESC_ALIGN_2 (1 << 16) /* All memory access in this area is aligned to their own size, or 2, whichever is smaller. */ -#define RETRO_MEMDESC_ALIGN_4 (2 << 16) -#define RETRO_MEMDESC_ALIGN_8 (3 << 16) -#define RETRO_MEMDESC_MINSIZE_2 (1 << 24) /* All memory in this region is accessed at least 2 bytes at the time. */ -#define RETRO_MEMDESC_MINSIZE_4 (2 << 24) -#define RETRO_MEMDESC_MINSIZE_8 (3 << 24) +/* Serialized state is incomplete in some way. Set if serialization is + * usable in typical end-user cases but should not be relied upon to + * implement frame-sensitive frontend features such as netplay or + * rerecording. */ +#define RETRO_SERIALIZATION_QUIRK_INCOMPLETE (1 << 0) +/* The core must spend some time initializing before serialization is + * supported. retro_serialize() will initially fail; retro_unserialize() + * and retro_serialize_size() may or may not work correctly either. */ +#define RETRO_SERIALIZATION_QUIRK_MUST_INITIALIZE (1 << 1) +/* Serialization size may change within a session. */ +#define RETRO_SERIALIZATION_QUIRK_CORE_VARIABLE_SIZE (1 << 2) +/* Set by the frontend to acknowledge that it supports variable-sized + * states. */ +#define RETRO_SERIALIZATION_QUIRK_FRONT_VARIABLE_SIZE (1 << 3) +/* Serialized state can only be loaded during the same session. */ +#define RETRO_SERIALIZATION_QUIRK_SINGLE_SESSION (1 << 4) +/* Serialized state cannot be loaded on an architecture with a different + * endianness from the one it was saved on. */ +#define RETRO_SERIALIZATION_QUIRK_ENDIAN_DEPENDENT (1 << 5) +/* Serialized state cannot be loaded on a different platform from the one it + * was saved on for reasons other than endianness, such as word size + * dependence */ +#define RETRO_SERIALIZATION_QUIRK_PLATFORM_DEPENDENT (1 << 6) + +#define RETRO_MEMDESC_CONST (1 << 0) /* The frontend will never change this memory area once retro_load_game has returned. */ +#define RETRO_MEMDESC_BIGENDIAN (1 << 1) /* The memory area contains big endian data. Default is little endian. */ +#define RETRO_MEMDESC_SYSTEM_RAM (1 << 2) /* The memory area is system RAM. This is main RAM of the gaming system. */ +#define RETRO_MEMDESC_SAVE_RAM (1 << 3) /* The memory area is save RAM. This RAM is usually found on a game cartridge, backed up by a battery. */ +#define RETRO_MEMDESC_VIDEO_RAM (1 << 4) /* The memory area is video RAM (VRAM) */ +#define RETRO_MEMDESC_ALIGN_2 (1 << 16) /* All memory access in this area is aligned to their own size, or 2, whichever is smaller. */ +#define RETRO_MEMDESC_ALIGN_4 (2 << 16) +#define RETRO_MEMDESC_ALIGN_8 (3 << 16) +#define RETRO_MEMDESC_MINSIZE_2 (1 << 24) /* All memory in this region is accessed at least 2 bytes at the time. */ +#define RETRO_MEMDESC_MINSIZE_4 (2 << 24) +#define RETRO_MEMDESC_MINSIZE_8 (3 << 24) struct retro_memory_descriptor { uint64_t flags; /* Pointer to the start of the relevant ROM or RAM chip. - * It's strongly recommended to use 'offset' if possible, rather than + * It's strongly recommended to use 'offset' if possible, rather than * doing math on the pointer. * - * If the same byte is mapped my multiple descriptors, their descriptors + * If the same byte is mapped my multiple descriptors, their descriptors * must have the same pointer. - * If 'start' does not point to the first byte in the pointer, put the + * If 'start' does not point to the first byte in the pointer, put the * difference in 'offset' instead. * - * May be NULL if there's nothing usable here (e.g. hardware registers and + * May be NULL if there's nothing usable here (e.g. hardware registers and * open bus). No flags should be set if the pointer is NULL. * It's recommended to minimize the number of descriptors if possible, * but not mandatory. */ void *ptr; size_t offset; - /* This is the location in the emulated address space + /* This is the location in the emulated address space * where the mapping starts. */ size_t start; /* Which bits must be same as in 'start' for this mapping to apply. - * The first memory descriptor to claim a certain byte is the one + * The first memory descriptor to claim a certain byte is the one * that applies. * A bit which is set in 'start' must also be set in this. - * Can be zero, in which case each byte is assumed mapped exactly once. + * Can be zero, in which case each byte is assumed mapped exactly once. * In this case, 'len' must be a power of two. */ size_t select; - /* If this is nonzero, the set bits are assumed not connected to the + /* If this is nonzero, the set bits are assumed not connected to the * memory chip's address pins. */ size_t disconnect; /* This one tells the size of the current memory area. - * If, after start+disconnect are applied, the address is higher than + * If, after start+disconnect are applied, the address is higher than * this, the highest bit of the address is cleared. * * If the address is still too high, the next highest bit is cleared. - * Can be zero, in which case it's assumed to be infinite (as limited + * Can be zero, in which case it's assumed to be infinite (as limited * by 'select' and 'disconnect'). */ size_t len; - /* To go from emulated address to physical address, the following + /* To go from emulated address to physical address, the following * order applies: * Subtract 'start', pick off 'disconnect', apply 'len', add 'offset'. */ - /* The address space name must consist of only a-zA-Z0-9_-, + /* The address space name must consist of only a-zA-Z0-9_-, * should be as short as feasible (maximum length is 8 plus the NUL), - * and may not be any other address space plus one or more 0-9A-F + * and may not be any other address space plus one or more 0-9A-F * at the end. - * However, multiple memory descriptors for the same address space is - * allowed, and the address space name can be empty. NULL is treated + * However, multiple memory descriptors for the same address space is + * allowed, and the address space name can be empty. NULL is treated * as empty. * * Address space names are case sensitive, but avoid lowercase if possible. @@ -1052,11 +1590,11 @@ struct retro_memory_descriptor * 'a'+blank - valid ('a' is not in 0-9A-F) * 'a'+'A' - valid (neither is a prefix of each other) * 'AR'+blank - valid ('R' is not in 0-9A-F) - * 'ARB'+blank - valid (the B can't be part of the address either, because + * 'ARB'+blank - valid (the B can't be part of the address either, because * there is no namespace 'AR') - * blank+'B' - not valid, because it's ambigous which address space B1234 + * blank+'B' - not valid, because it's ambigous which address space B1234 * would refer to. - * The length can't be used for that purpose; the frontend may want + * The length can't be used for that purpose; the frontend may want * to append arbitrary data to an address, without a separator. */ const char *addrspace; @@ -1078,32 +1616,32 @@ struct retro_memory_descriptor * the most recent addition and continue on the next bit. * TODO: Can the above be optimized? Is "remove the lowest bit set in both * pointer and 'len'" equivalent? */ - + /* TODO: Some emulators (MAME?) emulate big endian systems by only accessing * the emulated memory in 32-bit chunks, native endian. But that's nothing * compared to Darek Mihocka * (section Emulation 103 - Nearly Free Byte Reversal) - he flips the ENTIRE * RAM backwards! I'll want to represent both of those, via some flags. - * + * * I suspect MAME either didn't think of that idea, or don't want the #ifdef. * Not sure which, nor do I really care. */ - + /* TODO: Some of those flags are unused and/or don't really make sense. Clean * them up. */ }; -/* The frontend may use the largest value of 'start'+'select' in a +/* The frontend may use the largest value of 'start'+'select' in a * certain namespace to infer the size of the address space. * - * If the address space is larger than that, a mapping with .ptr=NULL - * should be at the end of the array, with .select set to all ones for + * If the address space is larger than that, a mapping with .ptr=NULL + * should be at the end of the array, with .select set to all ones for * as long as the address space is big. * * Sample descriptors (minus .ptr, and RETRO_MEMFLAG_ on the flags): * SNES WRAM: * .start=0x7E0000, .len=0x20000 - * (Note that this must be mapped before the ROM in most cases; some of the - * ROM mappers + * (Note that this must be mapped before the ROM in most cases; some of the + * ROM mappers * try to claim $7E0000, or at least $7E8000.) * SNES SPC700 RAM: * .addrspace="S", .len=0x10000 @@ -1112,7 +1650,7 @@ struct retro_memory_descriptor * .flags=MIRROR, .start=0x800000, .select=0xC0E000, .len=0x2000 * SNES WRAM mirrors, alternate equivalent descriptor: * .flags=MIRROR, .select=0x40E000, .disconnect=~0x1FFF - * (Various similar constructions can be created by combining parts of + * (Various similar constructions can be created by combining parts of * the above two.) * SNES LoROM (512KB, mirrored a couple of times): * .flags=CONST, .start=0x008000, .select=0x408000, .disconnect=0x8000, .len=512*1024 @@ -1138,13 +1676,13 @@ struct retro_memory_map struct retro_controller_description { - /* Human-readable description of the controller. Even if using a generic - * input device type, this can be set to the particular device type the + /* Human-readable description of the controller. Even if using a generic + * input device type, this can be set to the particular device type the * core uses. */ const char *desc; - /* Device type passed to retro_set_controller_port_device(). If the device - * type is a sub-class of a generic input device type, use the + /* Device type passed to retro_set_controller_port_device(). If the device + * type is a sub-class of a generic input device type, use the * RETRO_DEVICE_SUBCLASS macro to create an ID. * * E.g. RETRO_DEVICE_SUBCLASS(RETRO_DEVICE_JOYPAD, 1). */ @@ -1162,8 +1700,8 @@ struct retro_subsystem_memory_info /* The extension associated with a memory type, e.g. "psram". */ const char *extension; - /* The memory type for retro_get_memory(). This should be at - * least 0x100 to avoid conflict with standardized + /* The memory type for retro_get_memory(). This should be at + * least 0x100 to avoid conflict with standardized * libretro memory types. */ unsigned type; }; @@ -1182,11 +1720,11 @@ struct retro_subsystem_rom_info /* Same definition as retro_get_system_info(). */ bool block_extract; - /* This is set if the content is required to load a game. + /* This is set if the content is required to load a game. * If this is set to false, a zeroed-out retro_game_info can be passed. */ bool required; - /* Content can have multiple associated persistent + /* Content can have multiple associated persistent * memory types (retro_get_memory()). */ const struct retro_subsystem_memory_info *memory; unsigned num_memory; @@ -1204,17 +1742,17 @@ struct retro_subsystem_info */ const char *ident; - /* Infos for each content file. The first entry is assumed to be the + /* Infos for each content file. The first entry is assumed to be the * "most significant" content for frontend purposes. - * E.g. with Super GameBoy, the first content should be the GameBoy ROM, + * E.g. with Super GameBoy, the first content should be the GameBoy ROM, * as it is the most "significant" content to a user. - * If a frontend creates new file paths based on the content used + * If a frontend creates new file paths based on the content used * (e.g. savestates), it should use the path for the first ROM to do so. */ const struct retro_subsystem_rom_info *roms; /* Number of content files associated with a subsystem. */ unsigned num_roms; - + /* The type passed to retro_load_game_special(). */ unsigned id; }; @@ -1225,13 +1763,13 @@ typedef void (RETRO_CALLCONV *retro_proc_address_t)(void); * (None here so far). * * Get a symbol from a libretro core. - * Cores should only return symbols which are actual + * Cores should only return symbols which are actual * extensions to the libretro API. * - * Frontends should not use this to obtain symbols to standard + * Frontends should not use this to obtain symbols to standard * libretro entry points (static linking or dlsym). * - * The symbol name must be equal to the function name, + * The symbol name must be equal to the function name, * e.g. if void retro_foo(void); exists, the symbol must be called "retro_foo". * The returned function pointer must be cast to the corresponding type. */ @@ -1284,6 +1822,8 @@ struct retro_log_callback #define RETRO_SIMD_VFPV4 (1 << 17) #define RETRO_SIMD_POPCNT (1 << 18) #define RETRO_SIMD_MOVBE (1 << 19) +#define RETRO_SIMD_CMOV (1 << 20) +#define RETRO_SIMD_ASIMD (1 << 21) typedef uint64_t retro_perf_tick_t; typedef int64_t retro_time_t; @@ -1304,7 +1844,7 @@ struct retro_perf_counter typedef retro_time_t (RETRO_CALLCONV *retro_perf_get_time_usec_t)(void); /* A simple counter. Usually nanoseconds, but can also be CPU cycles. - * Can be used directly if desired (when creating a more sophisticated + * Can be used directly if desired (when creating a more sophisticated * performance counter system). * */ typedef retro_perf_tick_t (RETRO_CALLCONV *retro_perf_get_counter_t)(void); @@ -1318,9 +1858,9 @@ typedef uint64_t (RETRO_CALLCONV *retro_get_cpu_features_t)(void); typedef void (RETRO_CALLCONV *retro_perf_log_t)(void); /* Register a performance counter. - * ident field must be set with a discrete value and other values in + * ident field must be set with a discrete value and other values in * retro_perf_counter must be 0. - * Registering can be called multiple times. To avoid calling to + * Registering can be called multiple times. To avoid calling to * frontend redundantly, you can check registered field first. */ typedef void (RETRO_CALLCONV *retro_perf_register_t)(struct retro_perf_counter *counter); @@ -1391,7 +1931,7 @@ enum retro_sensor_action #define RETRO_SENSOR_ACCELEROMETER_Y 1 #define RETRO_SENSOR_ACCELEROMETER_Z 2 -typedef bool (RETRO_CALLCONV *retro_set_sensor_state_t)(unsigned port, +typedef bool (RETRO_CALLCONV *retro_set_sensor_state_t)(unsigned port, enum retro_sensor_action action, unsigned rate); typedef float (RETRO_CALLCONV *retro_sensor_get_input_t)(unsigned port, unsigned id); @@ -1416,7 +1956,7 @@ typedef bool (RETRO_CALLCONV *retro_camera_start_t)(void); /* Stops the camera driver. Can only be called in retro_run(). */ typedef void (RETRO_CALLCONV *retro_camera_stop_t)(void); -/* Callback which signals when the camera driver is initialized +/* Callback which signals when the camera driver is initialized * and/or deinitialized. * retro_camera_start_t can be called in initialized callback. */ @@ -1426,36 +1966,36 @@ typedef void (RETRO_CALLCONV *retro_camera_lifetime_status_t)(void); * Width, height and pitch are similar to retro_video_refresh_t. * First pixel is top-left origin. */ -typedef void (RETRO_CALLCONV *retro_camera_frame_raw_framebuffer_t)(const uint32_t *buffer, +typedef void (RETRO_CALLCONV *retro_camera_frame_raw_framebuffer_t)(const uint32_t *buffer, unsigned width, unsigned height, size_t pitch); /* A callback for when OpenGL textures are used. * * texture_id is a texture owned by camera driver. - * Its state or content should be considered immutable, except for things like + * Its state or content should be considered immutable, except for things like * texture filtering and clamping. * * texture_target is the texture target for the GL texture. - * These can include e.g. GL_TEXTURE_2D, GL_TEXTURE_RECTANGLE, and possibly + * These can include e.g. GL_TEXTURE_2D, GL_TEXTURE_RECTANGLE, and possibly * more depending on extensions. * - * affine points to a packed 3x3 column-major matrix used to apply an affine + * affine points to a packed 3x3 column-major matrix used to apply an affine * transform to texture coordinates. (affine_matrix * vec3(coord_x, coord_y, 1.0)) - * After transform, normalized texture coord (0, 0) should be bottom-left + * After transform, normalized texture coord (0, 0) should be bottom-left * and (1, 1) should be top-right (or (width, height) for RECTANGLE). * - * GL-specific typedefs are avoided here to avoid relying on gl.h in + * GL-specific typedefs are avoided here to avoid relying on gl.h in * the API definition. */ -typedef void (RETRO_CALLCONV *retro_camera_frame_opengl_texture_t)(unsigned texture_id, +typedef void (RETRO_CALLCONV *retro_camera_frame_opengl_texture_t)(unsigned texture_id, unsigned texture_target, const float *affine); struct retro_camera_callback { - /* Set by libretro core. + /* Set by libretro core. * Example bitmask: caps = (1 << RETRO_CAMERA_BUFFER_OPENGL_TEXTURE) | (1 << RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER). */ - uint64_t caps; + uint64_t caps; /* Desired resolution for camera. Is only used as a hint. */ unsigned width; @@ -1469,22 +2009,22 @@ struct retro_camera_callback retro_camera_frame_raw_framebuffer_t frame_raw_framebuffer; /* Set by libretro core if OpenGL texture callbacks will be used. */ - retro_camera_frame_opengl_texture_t frame_opengl_texture; + retro_camera_frame_opengl_texture_t frame_opengl_texture; - /* Set by libretro core. Called after camera driver is initialized and + /* Set by libretro core. Called after camera driver is initialized and * ready to be started. * Can be NULL, in which this callback is not called. */ retro_camera_lifetime_status_t initialized; - /* Set by libretro core. Called right before camera driver is + /* Set by libretro core. Called right before camera driver is * deinitialized. * Can be NULL, in which this callback is not called. */ retro_camera_lifetime_status_t deinitialized; }; -/* Sets the interval of time and/or distance at which to update/poll +/* Sets the interval of time and/or distance at which to update/poll * location-based data. * * To ensure compatibility with all location-based implementations, @@ -1497,20 +2037,20 @@ typedef void (RETRO_CALLCONV *retro_location_set_interval_t)(unsigned interval_m unsigned interval_distance); /* Start location services. The device will start listening for changes to the - * current location at regular intervals (which are defined with + * current location at regular intervals (which are defined with * retro_location_set_interval_t). */ typedef bool (RETRO_CALLCONV *retro_location_start_t)(void); -/* Stop location services. The device will stop listening for changes +/* Stop location services. The device will stop listening for changes * to the current location. */ typedef void (RETRO_CALLCONV *retro_location_stop_t)(void); -/* Get the position of the current location. Will set parameters to +/* Get the position of the current location. Will set parameters to * 0 if no new location update has happened since the last time. */ typedef bool (RETRO_CALLCONV *retro_location_get_position_t)(double *lat, double *lon, double *horiz_accuracy, double *vert_accuracy); -/* Callback which signals when the location driver is initialized +/* Callback which signals when the location driver is initialized * and/or deinitialized. * retro_location_start_t can be called in initialized callback. */ @@ -1535,14 +2075,14 @@ enum retro_rumble_effect RETRO_RUMBLE_DUMMY = INT_MAX }; -/* Sets rumble state for joypad plugged in port 'port'. +/* Sets rumble state for joypad plugged in port 'port'. * Rumble effects are controlled independently, * and setting e.g. strong rumble does not override weak rumble. * Strength has a range of [0, 0xffff]. * - * Returns true if rumble state request was honored. + * Returns true if rumble state request was honored. * Calling this before first retro_run() is likely to return false. */ -typedef bool (RETRO_CALLCONV *retro_set_rumble_state_t)(unsigned port, +typedef bool (RETRO_CALLCONV *retro_set_rumble_state_t)(unsigned port, enum retro_rumble_effect effect, uint16_t strength); struct retro_rumble_interface @@ -1553,10 +2093,10 @@ struct retro_rumble_interface /* Notifies libretro that audio data should be written. */ typedef void (RETRO_CALLCONV *retro_audio_callback_t)(void); -/* True: Audio driver in frontend is active, and callback is +/* True: Audio driver in frontend is active, and callback is * expected to be called regularily. - * False: Audio driver in frontend is paused or inactive. - * Audio callback will not be called until set_state has been + * False: Audio driver in frontend is paused or inactive. + * Audio callback will not be called until set_state has been * called with true. * Initial state is false (inactive). */ @@ -1568,11 +2108,11 @@ struct retro_audio_callback retro_audio_set_state_callback_t set_state; }; -/* Notifies a libretro core of time spent since last invocation +/* Notifies a libretro core of time spent since last invocation * of retro_run() in microseconds. * * It will be called right before retro_run() every frame. - * The frontend can tamper with timing to support cases like + * The frontend can tamper with timing to support cases like * fast-forward, slow-motion and framestepping. * * In those scenarios the reference frame time value will be used. */ @@ -1581,8 +2121,8 @@ typedef void (RETRO_CALLCONV *retro_frame_time_callback_t)(retro_usec_t usec); struct retro_frame_time_callback { retro_frame_time_callback_t callback; - /* Represents the time of one frame. It is computed as - * 1000000 / fps, but the implementation will resolve the + /* Represents the time of one frame. It is computed as + * 1000000 / fps, but the implementation will resolve the * rounding to ensure that framestepping, etc is exact. */ retro_usec_t reference; }; @@ -1598,7 +2138,7 @@ struct retro_frame_time_callback * it should implement context_destroy callback. * If called, all GPU resources must be reinitialized. * Usually called when frontend reinits video driver. - * Also called first time video driver is initialized, + * Also called first time video driver is initialized, * allowing libretro core to initialize resources. */ typedef void (RETRO_CALLCONV *retro_hw_context_reset_t)(void); @@ -1615,7 +2155,7 @@ enum retro_hw_context_type { RETRO_HW_CONTEXT_NONE = 0, /* OpenGL 2.x. Driver can choose to use latest compatibility context. */ - RETRO_HW_CONTEXT_OPENGL = 1, + RETRO_HW_CONTEXT_OPENGL = 1, /* OpenGL ES 2.0. */ RETRO_HW_CONTEXT_OPENGLES2 = 2, /* Modern desktop core GL context. Use version_major/ @@ -1630,6 +2170,10 @@ enum retro_hw_context_type /* Vulkan, see RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE. */ RETRO_HW_CONTEXT_VULKAN = 6, + /* Direct3D, set version_major to select the type of interface + * returned by RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE */ + RETRO_HW_CONTEXT_DIRECT3D = 7, + RETRO_HW_CONTEXT_DUMMY = INT_MAX }; @@ -1641,10 +2185,10 @@ struct retro_hw_render_callback /* Called when a context has been created or when it has been reset. * An OpenGL context is only valid after context_reset() has been called. * - * When context_reset is called, OpenGL resources in the libretro + * When context_reset is called, OpenGL resources in the libretro * implementation are guaranteed to be invalid. * - * It is possible that context_reset is called multiple times during an + * It is possible that context_reset is called multiple times during an * application lifecycle. * If context_reset is called without any notification (context_destroy), * the OpenGL context was lost and resources should just be recreated @@ -1657,7 +2201,8 @@ struct retro_hw_render_callback * be providing preallocated framebuffers. */ retro_hw_get_current_framebuffer_t get_current_framebuffer; - /* Set by frontend. */ + /* Set by frontend. + * Can return all relevant functions, including glClear on Windows. */ retro_hw_get_proc_address_t get_proc_address; /* Set if render buffers should have depth component attached. @@ -1668,48 +2213,48 @@ struct retro_hw_render_callback * TODO: Obsolete. */ bool stencil; - /* If depth and stencil are true, a packed 24/8 buffer will be added. + /* If depth and stencil are true, a packed 24/8 buffer will be added. * Only attaching stencil is invalid and will be ignored. */ - /* Use conventional bottom-left origin convention. If false, + /* Use conventional bottom-left origin convention. If false, * standard libretro top-left origin semantics are used. * TODO: Move to GL specific interface. */ bool bottom_left_origin; - + /* Major version number for core GL context or GLES 3.1+. */ unsigned version_major; /* Minor version number for core GL context or GLES 3.1+. */ unsigned version_minor; - /* If this is true, the frontend will go very far to avoid + /* If this is true, the frontend will go very far to avoid * resetting context in scenarios like toggling fullscreen, etc. * TODO: Obsolete? Maybe frontend should just always assume this ... */ bool cache_context; - /* The reset callback might still be called in extreme situations + /* The reset callback might still be called in extreme situations * such as if the context is lost beyond recovery. * - * For optimal stability, set this to false, and allow context to be + * For optimal stability, set this to false, and allow context to be * reset at any time. */ - - /* A callback to be called before the context is destroyed in a + + /* A callback to be called before the context is destroyed in a * controlled way by the frontend. */ retro_hw_context_reset_t context_destroy; /* OpenGL resources can be deinitialized cleanly at this step. - * context_destroy can be set to NULL, in which resources will + * context_destroy can be set to NULL, in which resources will * just be destroyed without any notification. * - * Even when context_destroy is non-NULL, it is possible that + * Even when context_destroy is non-NULL, it is possible that * context_reset is called without any destroy notification. - * This happens if context is lost by external factors (such as + * This happens if context is lost by external factors (such as * notified by GL_ARB_robustness). * * In this case, the context is assumed to be already dead, - * and the libretro implementation must not try to free any OpenGL + * and the libretro implementation must not try to free any OpenGL * resources in the subsequent context_reset. */ @@ -1717,7 +2262,7 @@ struct retro_hw_render_callback bool debug_context; }; -/* Callback type passed in RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. +/* Callback type passed in RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. * Called by the frontend in response to keyboard events. * down is set if the key is being pressed, or false if it is being released. * keycode is the RETROK value of the char. @@ -1725,16 +2270,16 @@ struct retro_hw_render_callback * key_modifiers is a set of RETROKMOD values or'ed together. * * The pressed/keycode state can be indepedent of the character. - * It is also possible that multiple characters are generated from a + * It is also possible that multiple characters are generated from a * single keypress. * Keycode events should be treated separately from character events. * However, when possible, the frontend should try to synchronize these. * If only a character is posted, keycode should be RETROK_UNKNOWN. * - * Similarily if only a keycode event is generated with no corresponding + * Similarily if only a keycode event is generated with no corresponding * character, character should be 0. */ -typedef void (RETRO_CALLCONV *retro_keyboard_event_t)(bool down, unsigned keycode, +typedef void (RETRO_CALLCONV *retro_keyboard_event_t)(bool down, unsigned keycode, uint32_t character, uint16_t key_modifiers); struct retro_keyboard_callback @@ -1743,15 +2288,15 @@ struct retro_keyboard_callback }; /* Callbacks for RETRO_ENVIRONMENT_SET_DISK_CONTROL_INTERFACE. - * Should be set for implementations which can swap out multiple disk + * Should be set for implementations which can swap out multiple disk * images in runtime. * * If the implementation can do this automatically, it should strive to do so. * However, there are cases where the user must manually do so. * - * Overview: To swap a disk image, eject the disk image with + * Overview: To swap a disk image, eject the disk image with * set_eject_state(true). - * Set the disk index with set_image_index(index). Insert the disk again + * Set the disk index with set_image_index(index). Insert the disk again * with set_eject_state(false). */ @@ -1769,7 +2314,7 @@ typedef bool (RETRO_CALLCONV *retro_get_eject_state_t)(void); typedef unsigned (RETRO_CALLCONV *retro_get_image_index_t)(void); /* Sets image index. Can only be called when disk is ejected. - * The implementation supports setting "no disk" by using an + * The implementation supports setting "no disk" by using an * index >= get_num_images(). */ typedef bool (RETRO_CALLCONV *retro_set_image_index_t)(unsigned index); @@ -1783,11 +2328,11 @@ struct retro_game_info; * Arguments to pass in info have same requirements as retro_load_game(). * Virtual disk tray must be ejected when calling this. * - * Replacing a disk image with info = NULL will remove the disk image + * Replacing a disk image with info = NULL will remove the disk image * from the internal list. * As a result, calls to get_image_index() can change. * - * E.g. replace_image_index(1, NULL), and previous get_image_index() + * E.g. replace_image_index(1, NULL), and previous get_image_index() * returned 4 before. * Index 1 will be removed, and the new index is 3. */ @@ -1796,7 +2341,7 @@ typedef bool (RETRO_CALLCONV *retro_replace_image_index_t)(unsigned index, /* Adds a new valid index (get_num_images()) to the internal disk list. * This will increment subsequent return values from get_num_images() by 1. - * This image index cannot be used until a disk image has been set + * This image index cannot be used until a disk image has been set * with replace_image_index. */ typedef bool (RETRO_CALLCONV *retro_add_image_index_t)(void); @@ -1827,7 +2372,7 @@ enum retro_pixel_format /* RGB565, native endian. * This pixel format is the recommended format to use if a 15/16-bit - * format is desired as it is the pixel format that is typically + * format is desired as it is the pixel format that is typically * available on a wide range of low-power devices. * * It is also natively supported in APIs like OpenGL ES. */ @@ -1857,43 +2402,52 @@ struct retro_input_descriptor /* Human readable description for parameters. * The pointer must remain valid until * retro_unload_game() is called. */ - const char *description; + const char *description; }; struct retro_system_info { - /* All pointers are owned by libretro implementation, and pointers must + /* All pointers are owned by libretro implementation, and pointers must * remain valid until retro_deinit() is called. */ - const char *library_name; /* Descriptive name of library. Should not + const char *library_name; /* Descriptive name of library. Should not * contain any version numbers, etc. */ const char *library_version; /* Descriptive version of core. */ - const char *valid_extensions; /* A string listing probably content - * extensions the core will be able to + const char *valid_extensions; /* A string listing probably content + * extensions the core will be able to * load, separated with pipe. * I.e. "bin|rom|iso". - * Typically used for a GUI to filter + * Typically used for a GUI to filter * out extensions. */ - /* If true, retro_load_game() is guaranteed to provide a valid pathname - * in retro_game_info::path. - * ::data and ::size are both invalid. + /* Libretro cores that need to have direct access to their content + * files, including cores which use the path of the content files to + * determine the paths of other files, should set need_fullpath to true. * - * If false, ::data and ::size are guaranteed to be valid, but ::path - * might not be valid. + * Cores should strive for setting need_fullpath to false, + * as it allows the frontend to perform patching, etc. * - * This is typically set to true for libretro implementations that must - * load from file. - * Implementations should strive for setting this to false, as it allows - * the frontend to perform patching, etc. */ - bool need_fullpath; + * If need_fullpath is true and retro_load_game() is called: + * - retro_game_info::path is guaranteed to have a valid path + * - retro_game_info::data and retro_game_info::size are invalid + * + * If need_fullpath is false and retro_load_game() is called: + * - retro_game_info::path may be NULL + * - retro_game_info::data and retro_game_info::size are guaranteed + * to be valid + * + * See also: + * - RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY + * - RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY + */ + bool need_fullpath; - /* If true, the frontend is not allowed to extract any archives before + /* If true, the frontend is not allowed to extract any archives before * loading the real content. - * Necessary for certain libretro implementations that load games + * Necessary for certain libretro implementations that load games * from zipped archives. */ - bool block_extract; + bool block_extract; }; struct retro_game_geometry @@ -1925,27 +2479,99 @@ struct retro_system_av_info struct retro_variable { /* Variable to query in RETRO_ENVIRONMENT_GET_VARIABLE. - * If NULL, obtains the complete environment string if more + * If NULL, obtains the complete environment string if more * complex parsing is necessary. - * The environment string is formatted as key-value pairs + * The environment string is formatted as key-value pairs * delimited by semicolons as so: * "key1=value1;key2=value2;..." */ const char *key; - + /* Value to be obtained. If key does not exist, it is set to NULL. */ const char *value; }; +struct retro_core_option_display +{ + /* Variable to configure in RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY */ + const char *key; + + /* Specifies whether variable should be displayed + * when presenting core options to the user */ + bool visible; +}; + +/* Maximum number of values permitted for a core option + * > Note: We have to set a maximum value due the limitations + * of the C language - i.e. it is not possible to create an + * array of structs each containing a variable sized array, + * so the retro_core_option_definition values array must + * have a fixed size. The size limit of 128 is a balancing + * act - it needs to be large enough to support all 'sane' + * core options, but setting it too large may impact low memory + * platforms. In practise, if a core option has more than + * 128 values then the implementation is likely flawed. + * To quote the above API reference: + * "The number of possible options should be very limited + * i.e. it should be feasible to cycle through options + * without a keyboard." + */ +#define RETRO_NUM_CORE_OPTION_VALUES_MAX 128 + +struct retro_core_option_value +{ + /* Expected option value */ + const char *value; + + /* Human-readable value label. If NULL, value itself + * will be displayed by the frontend */ + const char *label; +}; + +struct retro_core_option_definition +{ + /* Variable to query in RETRO_ENVIRONMENT_GET_VARIABLE. */ + const char *key; + + /* Human-readable core option description (used as menu label) */ + const char *desc; + + /* Human-readable core option information (used as menu sublabel) */ + const char *info; + + /* Array of retro_core_option_value structs, terminated by NULL */ + struct retro_core_option_value values[RETRO_NUM_CORE_OPTION_VALUES_MAX]; + + /* Default core option value. Must match one of the values + * in the retro_core_option_value array, otherwise will be + * ignored */ + const char *default_value; +}; + +struct retro_core_options_intl +{ + /* Pointer to an array of retro_core_option_definition structs + * - US English implementation + * - Must point to a valid array */ + struct retro_core_option_definition *us; + + /* Pointer to an array of retro_core_option_definition structs + * - Implementation for current frontend language + * - May be NULL */ + struct retro_core_option_definition *local; +}; + struct retro_game_info { const char *path; /* Path to game, UTF-8 encoded. - * Usually used as a reference. - * May be NULL if rom was loaded from stdin - * or similar. - * retro_system_info::need_fullpath guaranteed + * Sometimes used as a reference for building other paths. + * May be NULL if game was loaded from stdin or similar, + * but in this case some cores will be unable to load `data`. + * So, it is preferable to fabricate something here instead + * of passing NULL, which will help more cores to succeed. + * retro_system_info::need_fullpath requires * that this path is valid. */ - const void *data; /* Memory buffer of loaded game. Will be NULL + const void *data; /* Memory buffer of loaded game. Will be NULL * if need_fullpath was set. */ size_t size; /* Size of memory buffer. */ const char *meta; /* String of implementation specific meta-data. */ @@ -1983,25 +2609,25 @@ struct retro_framebuffer /* Callbacks */ -/* Environment callback. Gives implementations a way of performing +/* Environment callback. Gives implementations a way of performing * uncommon tasks. Extensible. */ typedef bool (RETRO_CALLCONV *retro_environment_t)(unsigned cmd, void *data); -/* Render a frame. Pixel format is 15-bit 0RGB1555 native endian +/* Render a frame. Pixel format is 15-bit 0RGB1555 native endian * unless changed (see RETRO_ENVIRONMENT_SET_PIXEL_FORMAT). * * Width and height specify dimensions of buffer. * Pitch specifices length in bytes between two lines in buffer. * - * For performance reasons, it is highly recommended to have a frame + * For performance reasons, it is highly recommended to have a frame * that is packed in memory, i.e. pitch == width * byte_per_pixel. - * Certain graphic APIs, such as OpenGL ES, do not like textures + * Certain graphic APIs, such as OpenGL ES, do not like textures * that are not packed in memory. */ typedef void (RETRO_CALLCONV *retro_video_refresh_t)(const void *data, unsigned width, unsigned height, size_t pitch); -/* Renders a single audio frame. Should only be used if implementation +/* Renders a single audio frame. Should only be used if implementation * generates a single sample at a time. * Format is signed 16-bit native endian. */ @@ -2019,20 +2645,20 @@ typedef size_t (RETRO_CALLCONV *retro_audio_sample_batch_t)(const int16_t *data, /* Polls input. */ typedef void (RETRO_CALLCONV *retro_input_poll_t)(void); -/* Queries for input for player 'port'. device will be masked with +/* Queries for input for player 'port'. device will be masked with * RETRO_DEVICE_MASK. * - * Specialization of devices such as RETRO_DEVICE_JOYPAD_MULTITAP that + * Specialization of devices such as RETRO_DEVICE_JOYPAD_MULTITAP that * have been set with retro_set_controller_port_device() * will still use the higher level RETRO_DEVICE_JOYPAD to request input. */ -typedef int16_t (RETRO_CALLCONV *retro_input_state_t)(unsigned port, unsigned device, +typedef int16_t (RETRO_CALLCONV *retro_input_state_t)(unsigned port, unsigned device, unsigned index, unsigned id); -/* Sets callbacks. retro_set_environment() is guaranteed to be called +/* Sets callbacks. retro_set_environment() is guaranteed to be called * before retro_init(). * - * The rest of the set_* functions are guaranteed to have been called + * The rest of the set_* functions are guaranteed to have been called * before the first call to retro_run() is made. */ RETRO_API void retro_set_environment(retro_environment_t); RETRO_API void retro_set_video_refresh(retro_video_refresh_t); @@ -2049,27 +2675,33 @@ RETRO_API void retro_deinit(void); * when the API is revised. */ RETRO_API unsigned retro_api_version(void); -/* Gets statically known system info. Pointers provided in *info +/* Gets statically known system info. Pointers provided in *info * must be statically allocated. * Can be called at any time, even before retro_init(). */ RETRO_API void retro_get_system_info(struct retro_system_info *info); /* Gets information about system audio/video timings and geometry. * Can be called only after retro_load_game() has successfully completed. - * NOTE: The implementation of this function might not initialize every + * NOTE: The implementation of this function might not initialize every * variable if needed. - * E.g. geom.aspect_ratio might not be initialized if core doesn't + * E.g. geom.aspect_ratio might not be initialized if core doesn't * desire a particular aspect ratio. */ RETRO_API void retro_get_system_av_info(struct retro_system_av_info *info); /* Sets device to be used for player 'port'. - * By default, RETRO_DEVICE_JOYPAD is assumed to be plugged into all + * By default, RETRO_DEVICE_JOYPAD is assumed to be plugged into all * available ports. - * Setting a particular device type is not a guarantee that libretro cores - * will only poll input based on that particular device type. It is only a - * hint to the libretro core when a core cannot automatically detect the - * appropriate input device type on its own. It is also relevant when a - * core can change its behavior depending on device type. */ + * Setting a particular device type is not a guarantee that libretro cores + * will only poll input based on that particular device type. It is only a + * hint to the libretro core when a core cannot automatically detect the + * appropriate input device type on its own. It is also relevant when a + * core can change its behavior depending on device type. + * + * As part of the core's implementation of retro_set_controller_port_device, + * the core should call RETRO_ENVIRONMENT_SET_INPUT_DESCRIPTORS to notify the + * frontend if the descriptions for any controls have changed as a + * result of changing the device type. + */ RETRO_API void retro_set_controller_port_device(unsigned port, unsigned device); /* Resets the current game. */ @@ -2077,18 +2709,18 @@ RETRO_API void retro_reset(void); /* Runs the game for one video frame. * During retro_run(), input_poll callback must be called at least once. - * + * * If a frame is not rendered for reasons where a game "dropped" a frame, - * this still counts as a frame, and retro_run() should explicitly dupe + * this still counts as a frame, and retro_run() should explicitly dupe * a frame if GET_CAN_DUPE returns true. * In this case, the video callback can take a NULL argument for data. */ RETRO_API void retro_run(void); -/* Returns the amount of data the implementation requires to serialize +/* Returns the amount of data the implementation requires to serialize * internal state (save states). - * Between calls to retro_load_game() and retro_unload_game(), the - * returned size is never allowed to be larger than a previous returned + * Between calls to retro_load_game() and retro_unload_game(), the + * returned size is never allowed to be larger than a previous returned * value, to ensure that the frontend can allocate a save state buffer once. */ RETRO_API size_t retro_serialize_size(void); @@ -2101,7 +2733,9 @@ RETRO_API bool retro_unserialize(const void *data, size_t size); RETRO_API void retro_cheat_reset(void); RETRO_API void retro_cheat_set(unsigned index, bool enabled, const char *code); -/* Loads a game. */ +/* Loads a game. + * Return true to indicate successful loading and false to indicate load failure. + */ RETRO_API bool retro_load_game(const struct retro_game_info *game); /* Loads a "special" kind of game. Should not be used, @@ -2111,7 +2745,7 @@ RETRO_API bool retro_load_game_special( const struct retro_game_info *info, size_t num_info ); -/* Unloads a currently loaded game. */ +/* Unloads the currently loaded game. Called before retro_deinit(void). */ RETRO_API void retro_unload_game(void); /* Gets region of game. */ diff --git a/src/platform/libretro/libretro_core_options.h b/src/platform/libretro/libretro_core_options.h new file mode 100644 index 000000000..930e13239 --- /dev/null +++ b/src/platform/libretro/libretro_core_options.h @@ -0,0 +1,372 @@ +#ifndef LIBRETRO_CORE_OPTIONS_H__ +#define LIBRETRO_CORE_OPTIONS_H__ + +#include +#include + +#include "libretro.h" +#include "retro_inline.h" + +#ifndef HAVE_NO_LANGEXTRA +#include "libretro_core_options_intl.h" +#endif + +/* + ******************************** + * VERSION: 1.3 + ******************************** + * + * - 1.3: Move translations to libretro_core_options_intl.h + * - libretro_core_options_intl.h includes BOM and utf-8 + * fix for MSVC 2010-2013 + * - Added HAVE_NO_LANGEXTRA flag to disable translations + * on platforms/compilers without BOM support + * - 1.2: Use core options v1 interface when + * RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION is >= 1 + * (previously required RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION == 1) + * - 1.1: Support generation of core options v0 retro_core_option_value + * arrays containing options with a single value + * - 1.0: First commit +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + ******************************** + * Core Option Definitions + ******************************** +*/ + +/* RETRO_LANGUAGE_ENGLISH */ + +/* Default language: + * - All other languages must include the same keys and values + * - Will be used as a fallback in the event that frontend language + * is not available + * - Will be used as a fallback for any missing entries in + * frontend language definition */ + +struct retro_core_option_definition option_defs_us[] = { + { + "mgba_solar_sensor_level", + "Solar Sensor Level", + "Sets ambient sunlight intensity. Can be used by games that included a solar sensor in their cartridges, e.g: the Boktai series.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, + { + "mgba_allow_opposing_directions", + "Allow Opposing Directional Input", + "Enabling this will allow pressing / quickly alternating / holding both left and right (or up and down) directions at the same time. This may cause movement-based glitches.", + { + { "no", "disabled" }, + { "yes", "enabled" }, + { NULL, NULL }, + }, + "no" + }, + { + "mgba_gb_model", + "Game Boy Model (requires restart)", + "Runs loaded content with a specific Game Boy model. 'Autodetect' will select the most appropriate model for the current game.", + { + { "Autodetect", NULL }, + { "Game Boy", NULL }, + { "Super Game Boy", NULL }, + { "Game Boy Color", NULL }, + { "Game Boy Advance", NULL }, + { NULL, NULL }, + }, + "Autodetect" + }, + { + "mgba_use_bios", + "Use BIOS File if Found (requires restart)", + "Use official BIOS/bootloader for emulated hardware, if present in RetroArch's system directory.", + { + { "ON", NULL }, + { "OFF", NULL }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_skip_bios", + "Skip BIOS Intro (requires restart)", + "When using an official BIOS/bootloader, skip the start-up logo animation. This setting is ignored when 'Use BIOS File if Found' is disabled.", + { + { "OFF", NULL }, + { "ON", NULL }, + { NULL, NULL }, + }, + "OFF" + }, + { + "mgba_sgb_borders", + "Use Super Game Boy Borders (requires restart)", + "Display Super Game Boy borders when running Super Game Boy enhanced games.", + { + { "ON", NULL }, + { "OFF", NULL }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_idle_optimization", + "Idle Loop Removal", + "Reduce system load by optimizing so-called 'idle-loops' - sections in the code where nothing happens, but the CPU runs at full speed (like a car revving in neutral). Improves performance, and should be enabled on low-end hardware.", + { + { "Remove Known", NULL }, + { "Detect and Remove", NULL }, + { "Don't Remove", NULL }, + { NULL, NULL }, + }, + "Remove Known" + }, + { + "mgba_frameskip", + "Frameskip", + "Skip frames to improve performance at the expense of visual smoothness. Value set here is the number of frames omitted after a frame is rendered - i.e. '0' = 60fps, '1' = 30fps, '2' = 15fps, etc.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, +#if defined(COLOR_16_BIT) && defined(COLOR_5_6_5) + { + "mgba_color_correction", + "Color Correction", + "Adjusts output colors to match the display of real GBA/GBC hardware.", + { + { "OFF", NULL }, + { "GBA", "Game Boy Advance" }, + { "GBC", "Game Boy Color" }, + { "Auto", NULL }, + { NULL, NULL }, + }, + "OFF" + }, +#endif + { NULL, NULL, NULL, {{0}}, NULL }, +}; + +/* + ******************************** + * Language Mapping + ******************************** +*/ + +#ifndef HAVE_NO_LANGEXTRA +struct retro_core_option_definition *option_defs_intl[RETRO_LANGUAGE_LAST] = { + option_defs_us, /* RETRO_LANGUAGE_ENGLISH */ + NULL, /* RETRO_LANGUAGE_JAPANESE */ + NULL, /* RETRO_LANGUAGE_FRENCH */ + NULL, /* RETRO_LANGUAGE_SPANISH */ + NULL, /* RETRO_LANGUAGE_GERMAN */ + NULL, /* RETRO_LANGUAGE_ITALIAN */ + NULL, /* RETRO_LANGUAGE_DUTCH */ + NULL, /* RETRO_LANGUAGE_PORTUGUESE_BRAZIL */ + NULL, /* RETRO_LANGUAGE_PORTUGUESE_PORTUGAL */ + NULL, /* RETRO_LANGUAGE_RUSSIAN */ + NULL, /* RETRO_LANGUAGE_KOREAN */ + NULL, /* RETRO_LANGUAGE_CHINESE_TRADITIONAL */ + NULL, /* RETRO_LANGUAGE_CHINESE_SIMPLIFIED */ + NULL, /* RETRO_LANGUAGE_ESPERANTO */ + NULL, /* RETRO_LANGUAGE_POLISH */ + NULL, /* RETRO_LANGUAGE_VIETNAMESE */ + NULL, /* RETRO_LANGUAGE_ARABIC */ + NULL, /* RETRO_LANGUAGE_GREEK */ + option_defs_tr, /* RETRO_LANGUAGE_TURKISH */ +}; +#endif + +/* + ******************************** + * Functions + ******************************** +*/ + +/* Handles configuration/setting of core options. + * Should be called as early as possible - ideally inside + * retro_set_environment(), and no later than retro_load_game() + * > We place the function body in the header to avoid the + * necessity of adding more .c files (i.e. want this to + * be as painless as possible for core devs) + */ + +static INLINE void libretro_set_core_options(retro_environment_t environ_cb) +{ + unsigned version = 0; + + if (!environ_cb) + return; + + if (environ_cb(RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION, &version) && (version >= 1)) + { +#ifndef HAVE_NO_LANGEXTRA + struct retro_core_options_intl core_options_intl; + unsigned language = 0; + + core_options_intl.us = option_defs_us; + core_options_intl.local = NULL; + + if (environ_cb(RETRO_ENVIRONMENT_GET_LANGUAGE, &language) && + (language < RETRO_LANGUAGE_LAST) && (language != RETRO_LANGUAGE_ENGLISH)) + core_options_intl.local = option_defs_intl[language]; + + environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL, &core_options_intl); +#else + environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS, &option_defs_us); +#endif + } + else + { + size_t i; + size_t num_options = 0; + struct retro_variable *variables = NULL; + char **values_buf = NULL; + + /* Determine number of options */ + while (true) + { + if (option_defs_us[num_options].key) + num_options++; + else + break; + } + + /* Allocate arrays */ + variables = (struct retro_variable *)calloc(num_options + 1, sizeof(struct retro_variable)); + values_buf = (char **)calloc(num_options, sizeof(char *)); + + if (!variables || !values_buf) + goto error; + + /* Copy parameters from option_defs_us array */ + for (i = 0; i < num_options; i++) + { + const char *key = option_defs_us[i].key; + const char *desc = option_defs_us[i].desc; + const char *default_value = option_defs_us[i].default_value; + struct retro_core_option_value *values = option_defs_us[i].values; + size_t buf_len = 3; + size_t default_index = 0; + + values_buf[i] = NULL; + + if (desc) + { + size_t num_values = 0; + + /* Determine number of values */ + while (true) + { + if (values[num_values].value) + { + /* Check if this is the default value */ + if (default_value) + if (strcmp(values[num_values].value, default_value) == 0) + default_index = num_values; + + buf_len += strlen(values[num_values].value); + num_values++; + } + else + break; + } + + /* Build values string */ + if (num_values > 0) + { + size_t j; + + buf_len += num_values - 1; + buf_len += strlen(desc); + + values_buf[i] = (char *)calloc(buf_len, sizeof(char)); + if (!values_buf[i]) + goto error; + + strcpy(values_buf[i], desc); + strcat(values_buf[i], "; "); + + /* Default value goes first */ + strcat(values_buf[i], values[default_index].value); + + /* Add remaining values */ + for (j = 0; j < num_values; j++) + { + if (j != default_index) + { + strcat(values_buf[i], "|"); + strcat(values_buf[i], values[j].value); + } + } + } + } + + variables[i].key = key; + variables[i].value = values_buf[i]; + } + + /* Set variables */ + environ_cb(RETRO_ENVIRONMENT_SET_VARIABLES, variables); + +error: + + /* Clean up */ + if (values_buf) + { + for (i = 0; i < num_options; i++) + { + if (values_buf[i]) + { + free(values_buf[i]); + values_buf[i] = NULL; + } + } + + free(values_buf); + values_buf = NULL; + } + + if (variables) + { + free(variables); + variables = NULL; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/platform/libretro/libretro_core_options_intl.h b/src/platform/libretro/libretro_core_options_intl.h new file mode 100644 index 000000000..42fc012b3 --- /dev/null +++ b/src/platform/libretro/libretro_core_options_intl.h @@ -0,0 +1,209 @@ +#ifndef LIBRETRO_CORE_OPTIONS_INTL_H__ +#define LIBRETRO_CORE_OPTIONS_INTL_H__ + +#if defined(_MSC_VER) && (_MSC_VER >= 1500 && _MSC_VER < 1900) +/* https://support.microsoft.com/en-us/kb/980263 */ +#pragma execution_character_set("utf-8") +#pragma warning(disable:4566) +#endif + +#include "libretro.h" + +/* + ******************************** + * VERSION: 1.3 + ******************************** + * + * - 1.3: Move translations to libretro_core_options_intl.h + * - libretro_core_options_intl.h includes BOM and utf-8 + * fix for MSVC 2010-2013 + * - Added HAVE_NO_LANGEXTRA flag to disable translations + * on platforms/compilers without BOM support + * - 1.2: Use core options v1 interface when + * RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION is >= 1 + * (previously required RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION == 1) + * - 1.1: Support generation of core options v0 retro_core_option_value + * arrays containing options with a single value + * - 1.0: First commit +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + ******************************** + * Core Option Definitions + ******************************** +*/ + +/* RETRO_LANGUAGE_JAPANESE */ + +/* RETRO_LANGUAGE_FRENCH */ + +/* RETRO_LANGUAGE_SPANISH */ + +/* RETRO_LANGUAGE_GERMAN */ + +/* RETRO_LANGUAGE_ITALIAN */ + +/* RETRO_LANGUAGE_DUTCH */ + +/* RETRO_LANGUAGE_PORTUGUESE_BRAZIL */ + +/* RETRO_LANGUAGE_PORTUGUESE_PORTUGAL */ + +/* RETRO_LANGUAGE_RUSSIAN */ + +/* RETRO_LANGUAGE_KOREAN */ + +/* RETRO_LANGUAGE_CHINESE_TRADITIONAL */ + +/* RETRO_LANGUAGE_CHINESE_SIMPLIFIED */ + +/* RETRO_LANGUAGE_ESPERANTO */ + +/* RETRO_LANGUAGE_POLISH */ + +/* RETRO_LANGUAGE_VIETNAMESE */ + +/* RETRO_LANGUAGE_ARABIC */ + +/* RETRO_LANGUAGE_GREEK */ + +/* RETRO_LANGUAGE_TURKISH */ + +struct retro_core_option_definition option_defs_tr[] = { + { + "mgba_solar_sensor_level", + "Güneş Sensörü Seviyesi", + "Ortam güneş ışığının yoğunluğunu ayarlar. Boktai serisi, kartuşlarına güneş sensörü içeren oyunlar tarafından kullanılabilir.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, + { + "mgba_allow_opposing_directions", + "Karşı Yönlü Girdiye Çıkmaya İzin Ver", + "Bunu etkinleştirmek aynı anda hem sola hem de sağa (veya yukarı ve aşağı) yönlere basma / hızlı değiştirme / tutma imkanı sağlar. Bu harekete dayalı hatalara neden olabilir.", + { + { "no", "Devre dışı bırak" }, + { "yes", "Etkileştir" }, + { NULL, NULL }, + }, + "no" + }, + { + "mgba_gb_model", + "Game Boy Modeli (yeniden başlatma gerektirir)", + "Yüklenen içeriği belirli bir Game Boy modeliyle çalıştırır. 'Otomatik Tespit' mevcut oyun için en uygun modeli seçecektir.", + { + { "Autodetect", "Otomatik Tespit" }, + { "Game Boy", NULL }, + { "Super Game Boy", NULL }, + { "Game Boy Color", NULL }, + { "Game Boy Advance", NULL }, + { NULL, NULL }, + }, + "Autodetect" + }, + { + "mgba_use_bios", + "Bulunursa BIOS Dosyasını kullanın (yeniden başlatma gerektirir)", + "RetroArch'ın sistem dizininde varsa, öykünülmüş donanım için resmi BIOS/önyükleyici kullanır.", + { + { "ON", "AÇIK" }, + { "OFF", "KAPALI" }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_skip_bios", + "BIOS Girişini Atla (yeniden başlatma gerektirir)", + "Resmi bir BIOS / önyükleyici kullanırken, başlangıç logosu animasyonunu atlayın. Bu ayar, 'Bulunursa BIOS Dosyasını Kullan' devre dışı bırakıldığında geçersiz sayılır.", + { + { "OFF", "HAYIR" }, + { "ON", "AÇIK" }, + { NULL, NULL }, + }, + "OFF" + }, + { + "mgba_sgb_borders", + "Super Game Boy Sınırlarını kullanın (yeniden başlatma gerekir)", + "Super Game Boy gelişmiş oyunlarını çalıştırırken Super Game Boy sınırlarını görüntüleR.", + { + { "ON", "AÇIK" }, + { "OFF", "KAPALI" }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_idle_optimization", + "Boşta Döngü Kaldırma", + "'Boşta döngüler' denilen sistemi optimize ederek sistem yükünü azaltın - hiçbir şeyin olmadığı koddaki bölümler için, CPU tam hızda çalıştırır (boşa dönen bir araba gibi). Performansı arttırır ve düşük kaliteli donanımlarda etkinleştirilmesi gerekir.", + { + { "Remove Known", "Bilinenleri Kaldır" }, + { "Detect and Remove", "Algıla ve Kaldır" }, + { "Don't Remove", "Kaldırma" }, + { NULL, NULL }, + }, + "Remove Known" + }, + { + "mgba_frameskip", + "Kare atlama", + "Görsel pürüzsüzlük pahasına performansı artırmak için çerçeveleri atlayın. Burada ayarlanan değer, bir kare oluşturulduktan sonra atlanan kare sayısıdır - yani '0' = 60fps, '1' = 30fps, '2' = 15fps, vb.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, +#if defined(COLOR_16_BIT) && defined(COLOR_5_6_5) + { + "mgba_color_correction", + "Renk Düzeltmesi", + "Çıktı renklerini gerçek GBA / GBC donanımının görüntüsüyle eşleşecek şekilde ayarlar.", + { + { "OFF", "KAPALI" }, + { "GBA", "Game Boy Advance" }, + { "GBC", "Game Boy Color" }, + { "Auto", "Otomatik" }, + { NULL, NULL }, + }, + "OFF" + }, +#endif + { NULL, NULL, NULL, {{0}}, NULL }, +}; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/platform/libretro/retro_inline.h b/src/platform/libretro/retro_inline.h new file mode 100644 index 000000000..e4a21f6c4 --- /dev/null +++ b/src/platform/libretro/retro_inline.h @@ -0,0 +1,39 @@ +/* Copyright (C) 2010-2018 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (retro_inline.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __LIBRETRO_SDK_INLINE_H +#define __LIBRETRO_SDK_INLINE_H + +#ifndef INLINE + +#if defined(_WIN32) || defined(__INTEL_COMPILER) +#define INLINE __inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901L +#define INLINE inline +#elif defined(__GNUC__) +#define INLINE __inline__ +#else +#define INLINE +#endif + +#endif +#endif From 2f68ea1b48da14a1a0be06e84fd9a898d93bcdfd Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Mon, 13 Jan 2020 18:01:59 -0800 Subject: [PATCH 08/20] Wii: Fix game fast-forwarding after slowing down --- CHANGES | 1 + src/platform/wii/main.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 223b90ad1..fa6b16d6f 100644 --- a/CHANGES +++ b/CHANGES @@ -112,6 +112,7 @@ Other fixes: - Qt: Fix inability to clear default keybindings - Qt: Release held actions if they get rebound - Vita: Fix analog controls (fixes mgba.io/i/1554) + - Wii: Fix game fast-forwarding after slowing down Misc: - GB Memory: Support manual SRAM editing (fixes mgba.io/i/1580) - GBA Audio: Redo channel 4 batching for GBA only diff --git a/src/platform/wii/main.c b/src/platform/wii/main.c index 20cd2445b..d06a39e61 100644 --- a/src/platform/wii/main.c +++ b/src/platform/wii/main.c @@ -648,6 +648,8 @@ static void _drawStart(void) { VIDEO_WaitVSync(); } referenceRetraceCount = retraceCount; + } else if (frameLimiter && referenceRetraceCount < retraceCount - 1) { + referenceRetraceCount = retraceCount - 1; } _CPU_ISR_Restore(level); From d4e30b74ff3d34352ca86094bb1a4f97ccf12a8f Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Mon, 13 Jan 2020 19:24:57 -0800 Subject: [PATCH 09/20] Wii: Improve audio buffering (fixes #1617) --- CHANGES | 1 + src/platform/wii/main.c | 55 ++++++++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index fa6b16d6f..64b749dce 100644 --- a/CHANGES +++ b/CHANGES @@ -113,6 +113,7 @@ Other fixes: - Qt: Release held actions if they get rebound - Vita: Fix analog controls (fixes mgba.io/i/1554) - Wii: Fix game fast-forwarding after slowing down + - Wii: Improve audio buffering (fixes mgba.io/i/1617) Misc: - GB Memory: Support manual SRAM editing (fixes mgba.io/i/1580) - GBA Audio: Redo channel 4 batching for GBA only diff --git a/src/platform/wii/main.c b/src/platform/wii/main.c index d06a39e61..63d48214b 100644 --- a/src/platform/wii/main.c +++ b/src/platform/wii/main.c @@ -64,7 +64,8 @@ static enum VideoMode { VM_MAX } videoMode = VM_AUTODETECT; -#define SAMPLES 1024 +#define SAMPLES 512 +#define BUFFERS 8 #define GUI_SCALE 1.35f #define GUI_SCALE_240p 2.0f @@ -131,9 +132,12 @@ size_t romBufferSize; static void* framebuffer[2] = { 0, 0 }; static int whichFb = 0; -static struct GBAStereoSample audioBuffer[3][SAMPLES] __attribute__((__aligned__(32))); -static volatile size_t audioBufferSize = 0; +static struct AudioBuffer { + struct GBAStereoSample samples[SAMPLES] __attribute__((__aligned__(32))); + volatile size_t size; +} audioBuffer[BUFFERS] = {0}; static volatile int currentAudioBuffer = 0; +static volatile int nextAudioBuffer = 0; static double audioSampleRate = 60.0 / 1.001; static struct GUIFont* font; @@ -610,32 +614,43 @@ int main(int argc, char* argv[]) { } static void _audioDMA(void) { - if (!audioBufferSize) { + struct AudioBuffer* buffer = &audioBuffer[currentAudioBuffer]; + if (buffer->size != SAMPLES) { return; } - DCFlushRange(audioBuffer[currentAudioBuffer], audioBufferSize * sizeof(struct GBAStereoSample)); - AUDIO_InitDMA((u32) audioBuffer[currentAudioBuffer], audioBufferSize * sizeof(struct GBAStereoSample)); - currentAudioBuffer = (currentAudioBuffer + 1) % 3; - audioBufferSize = 0; + DCFlushRange(buffer->samples, SAMPLES * sizeof(struct GBAStereoSample)); + AUDIO_InitDMA((u32) buffer->samples, SAMPLES * sizeof(struct GBAStereoSample)); + buffer->size = 0; + currentAudioBuffer = (currentAudioBuffer + 1) % BUFFERS; } static void _postAudioBuffer(struct mAVStream* stream, blip_t* left, blip_t* right) { UNUSED(stream); + + u32 level = 0; + _CPU_ISR_Disable(level); + struct AudioBuffer* buffer = &audioBuffer[nextAudioBuffer]; int available = blip_samples_avail(left); - if (available + audioBufferSize > SAMPLES) { - available = SAMPLES - audioBufferSize; + if (available + buffer->size > SAMPLES) { + available = SAMPLES - buffer->size; } - available &= ~((32 / sizeof(struct GBAStereoSample)) - 1); // Force align to 32 bytes if (available > 0) { // These appear to be reversed for AUDIO_InitDMA - blip_read_samples(left, &audioBuffer[currentAudioBuffer][audioBufferSize].right, available, true); - blip_read_samples(right, &audioBuffer[currentAudioBuffer][audioBufferSize].left, available, true); - audioBufferSize += available; + blip_read_samples(left, &buffer->samples[buffer->size].right, available, true); + blip_read_samples(right, &buffer->samples[buffer->size].left, available, true); + buffer->size += available; } - if (audioBufferSize == SAMPLES && !AUDIO_GetDMAEnableFlag()) { - _audioDMA(); - AUDIO_StartDMA(); + if (buffer->size == SAMPLES) { + int next = (nextAudioBuffer + 1) % BUFFERS; + if ((currentAudioBuffer + BUFFERS - next) % BUFFERS != 1) { + nextAudioBuffer = next; + } + if (!AUDIO_GetDMAEnableFlag()) { + _audioDMA(); + AUDIO_StartDMA(); + } } + _CPU_ISR_Restore(level); } static void _drawStart(void) { @@ -798,6 +813,12 @@ void _setup(struct mGUIRunner* runner) { outputBuffer = memalign(32, TEX_W * TEX_H * BYTES_PER_PIXEL); runner->core->setVideoBuffer(runner->core, outputBuffer, TEX_W); + nextAudioBuffer = 0; + currentAudioBuffer = 0; + int i; + for (i = 0; i < BUFFERS; ++i) { + audioBuffer[i].size = 0; + } runner->core->setAudioBufferSize(runner->core, SAMPLES); double ratio = GBAAudioCalculateRatio(1, audioSampleRate, 1); From ab1d825e3038bd4aac252504c757a11be2435aff Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Tue, 14 Jan 2020 23:23:53 -0800 Subject: [PATCH 10/20] GBA Video: Slightly optimize frame compositing --- src/gba/renderers/gl.c | 46 +++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/gba/renderers/gl.c b/src/gba/renderers/gl.c index 398f14a57..f9f2c4f33 100644 --- a/src/gba/renderers/gl.c +++ b/src/gba/renderers/gl.c @@ -585,9 +585,6 @@ static const char* const _finalize = "out vec4 color;\n" "void composite(vec4 pixel, ivec4 flags, inout vec4 topPixel, inout ivec4 topFlags, inout vec4 bottomPixel, inout ivec4 bottomFlags) {\n" - " if (pixel.a == 0.) {\n" - " return;\n" - " }\n" " if (flags.x >= topFlags.x) {\n" " if (flags.x >= bottomFlags.x) {\n" " return;\n" @@ -607,32 +604,43 @@ static const char* const _finalize = " vec4 bottomPixel = topPixel;\n" " ivec4 topFlags = ivec4(texelFetch(backdropFlags, ivec2(0, texCoord.y), 0));\n" " ivec4 bottomFlags = topFlags;\n" - " ivec4 windowFlags = texelFetch(window, ivec2(texCoord * float(scale)), 0);\n" + " ivec2 coord = ivec2(texCoord * float(scale));\n" + " ivec4 windowFlags = texelFetch(window, coord, 0);\n" " int layerWindow = windowFlags.x;\n" " if ((layerWindow & 16) != 0) {\n" - " vec4 pix = texelFetch(layers[4], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[4], ivec2(texCoord * float(scale)), 0));\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[4], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[4], coord, 0));\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 1) != 0) {\n" - " vec4 pix = texelFetch(layers[0], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[0], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[0], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[0], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 2) != 0) {\n" - " vec4 pix = texelFetch(layers[1], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[1], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[1], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[1], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 4) != 0) {\n" - " vec4 pix = texelFetch(layers[2], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[2], ivec2(texCoord * float(scale)), 0).xyz.xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[2], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[2], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 8) != 0) {\n" - " vec4 pix = texelFetch(layers[3], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[3], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[3], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[3], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 32) == 0) {\n" " topFlags.y &= ~1;\n" From 0e49b2d3919428b219af8f9196ac29ec6c1f4c4c Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Wed, 15 Jan 2020 18:52:35 -0800 Subject: [PATCH 11/20] GBA Timers: Improve timer startup emulation --- CHANGES | 1 + src/gba/io.c | 8 ++++---- src/gba/timer.c | 13 +++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index 64b749dce..4e1a4edf5 100644 --- a/CHANGES +++ b/CHANGES @@ -92,6 +92,7 @@ Emulation fixes: - ARM: Fix stepping when events are pending - GBA DMA: Fix case where DMAs could get misaligned (fixes mgba.io/i/1092) - GBA Memory: Fix open bus from IWRAM (fixes mgba.io/i/1575) + - GBA Timers: Improve timer startup emulation - GBA Video: Fix OpenGL renderer 512x512 backgrounds (fixes mgba.io/i/1572) - GBA Video: Fix BLDY for semitransparent sprite on non-target-2 backgrounds - GBA Video: Fix effects blending improperly in some non-last windows diff --git a/src/gba/io.c b/src/gba/io.c index 579afacec..2b74dd99b 100644 --- a/src/gba/io.c +++ b/src/gba/io.c @@ -712,16 +712,16 @@ uint16_t GBAIORead(struct GBA* gba, uint32_t address) { switch (address) { // Reading this takes two cycles (1N+1I), so let's remove them preemptively case REG_TM0CNT_LO: - GBATimerUpdateRegister(gba, 0, 4); + GBATimerUpdateRegister(gba, 0, 2); break; case REG_TM1CNT_LO: - GBATimerUpdateRegister(gba, 1, 4); + GBATimerUpdateRegister(gba, 1, 2); break; case REG_TM2CNT_LO: - GBATimerUpdateRegister(gba, 2, 4); + GBATimerUpdateRegister(gba, 2, 2); break; case REG_TM3CNT_LO: - GBATimerUpdateRegister(gba, 3, 4); + GBATimerUpdateRegister(gba, 3, 2); break; case REG_KEYINPUT: diff --git a/src/gba/timer.c b/src/gba/timer.c index 9e097eff6..512a1d05f 100644 --- a/src/gba/timer.c +++ b/src/gba/timer.c @@ -8,9 +8,6 @@ #include #include -#define TIMER_RELOAD_DELAY 0 -#define TIMER_STARTUP_DELAY 2 - #define REG_TMCNT_LO(X) (REG_TM0CNT_LO + ((X) << 2)) static void GBATimerUpdate(struct GBA* gba, int timerId, uint32_t cyclesLate) { @@ -18,7 +15,7 @@ static void GBATimerUpdate(struct GBA* gba, int timerId, uint32_t cyclesLate) { if (GBATimerFlagsIsCountUp(timer->flags)) { gba->memory.io[REG_TMCNT_LO(timerId) >> 1] = timer->reload; } else { - GBATimerUpdateRegister(gba, timerId, TIMER_RELOAD_DELAY + cyclesLate); + GBATimerUpdateRegister(gba, timerId, cyclesLate); } if (GBATimerFlagsIsDoIrq(timer->flags)) { @@ -150,14 +147,14 @@ void GBATimerWriteTMCNT_HI(struct GBA* gba, int timer, uint16_t control) { mTimingDeschedule(&gba->timing, ¤tTimer->event); gba->memory.io[REG_TMCNT_LO(timer) >> 1] = currentTimer->reload; int32_t tickMask = (1 << prescaleBits) - 1; - currentTimer->lastEvent = (mTimingCurrentTime(&gba->timing) - TIMER_STARTUP_DELAY) & ~tickMask; - GBATimerUpdateRegister(gba, timer, TIMER_STARTUP_DELAY); + currentTimer->lastEvent = mTimingCurrentTime(&gba->timing) & ~tickMask; + GBATimerUpdateRegister(gba, timer, 0); } else if (wasEnabled && !GBATimerFlagsIsEnable(currentTimer->flags)) { mTimingDeschedule(&gba->timing, ¤tTimer->event); } else if (GBATimerFlagsIsEnable(currentTimer->flags) && GBATimerFlagsGetPrescaleBits(currentTimer->flags) != oldPrescale && !GBATimerFlagsIsCountUp(currentTimer->flags)) { mTimingDeschedule(&gba->timing, ¤tTimer->event); int32_t tickMask = (1 << prescaleBits) - 1; - currentTimer->lastEvent = (mTimingCurrentTime(&gba->timing) - TIMER_STARTUP_DELAY) & ~tickMask; - GBATimerUpdateRegister(gba, timer, TIMER_STARTUP_DELAY); + currentTimer->lastEvent = mTimingCurrentTime(&gba->timing) & ~tickMask; + GBATimerUpdateRegister(gba, timer, 0); } } From 8a6cf8dc9c00082e9917b4777f949de0d35ce710 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 16 Jan 2020 19:21:18 -0800 Subject: [PATCH 12/20] GBA Memory: Improve gamepak prefetch timing --- CHANGES | 1 + src/gba/memory.c | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 4e1a4edf5..ecf6a8af7 100644 --- a/CHANGES +++ b/CHANGES @@ -92,6 +92,7 @@ Emulation fixes: - ARM: Fix stepping when events are pending - GBA DMA: Fix case where DMAs could get misaligned (fixes mgba.io/i/1092) - GBA Memory: Fix open bus from IWRAM (fixes mgba.io/i/1575) + - GBA Memory: Improve gamepak prefetch timing - GBA Timers: Improve timer startup emulation - GBA Video: Fix OpenGL renderer 512x512 backgrounds (fixes mgba.io/i/1572) - GBA Video: Fix BLDY for semitransparent sprite on non-target-2 backgrounds diff --git a/src/gba/memory.c b/src/gba/memory.c index 091bfa1e1..10cdd777c 100644 --- a/src/gba/memory.c +++ b/src/gba/memory.c @@ -1622,17 +1622,19 @@ int32_t GBAMemoryStall(struct ARMCore* cpu, int32_t wait) { maxLoads -= previousLoads; } - int32_t s = cpu->memory.activeSeqCycles16 + 1; + int32_t s = cpu->memory.activeSeqCycles16; int32_t n2s = cpu->memory.activeNonseqCycles16 - cpu->memory.activeSeqCycles16 + 1; // Figure out how many sequential loads we can jam in - int32_t stall = s; + int32_t stall = s + 1; int32_t loads = 1; while (stall < wait && loads < maxLoads) { stall += s; ++loads; } + memory->lastPrefetchedPc = cpu->gprs[ARM_PC] + WORD_SIZE_THUMB * (loads + previousLoads - 1); + if (stall > wait) { // The wait cannot take less time than the prefetch stalls wait = stall; @@ -1641,10 +1643,9 @@ int32_t GBAMemoryStall(struct ARMCore* cpu, int32_t wait) { // This instruction used to have an N, convert it to an S. wait -= n2s; - memory->lastPrefetchedPc = cpu->gprs[ARM_PC] + WORD_SIZE_THUMB * (loads + previousLoads - 1); - // The next |loads|S waitstates disappear entirely, so long as they're all in a row - cpu->cycles -= (s - 1) * loads; + wait -= stall - 1; + return wait; } From dab12cf5c674542cae0db7708c333035255fbc65 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 16 Jan 2020 19:43:28 -0800 Subject: [PATCH 13/20] All: Fix several warnings --- src/arm/debugger/cli-debugger.c | 2 +- src/debugger/cli-debugger.c | 1 + src/gb/extra/proxy.c | 4 ++-- src/gba/core.c | 11 +++++++---- src/gba/renderers/gl.c | 7 ++++--- src/sm83/debugger/cli-debugger.c | 4 ++-- src/util/gui.c | 6 ++++-- src/util/hash.c | 16 +++++++++++----- src/util/string.c | 14 +++++++------- src/util/vfs.c | 2 +- 10 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/arm/debugger/cli-debugger.c b/src/arm/debugger/cli-debugger.c index 3dc87c4fc..c393f8c1e 100644 --- a/src/arm/debugger/cli-debugger.c +++ b/src/arm/debugger/cli-debugger.c @@ -86,7 +86,7 @@ static void _disassembleMode(struct CLIDebugger* debugger, struct CLIDebugVector size = 1; } else { size = dv->intValue; - dv = dv->next; // TODO: Check for excess args + // TODO: Check for excess args } int i; diff --git a/src/debugger/cli-debugger.c b/src/debugger/cli-debugger.c index 474909270..c3e604a07 100644 --- a/src/debugger/cli-debugger.c +++ b/src/debugger/cli-debugger.c @@ -883,6 +883,7 @@ static int _tryCommands(struct CLIDebugger* debugger, struct CLIDebuggerCommandS if (dvNext->type == CLIDV_ERROR_TYPE) { debugger->backend->printf(debugger->backend, "Parse error\n"); _DVFree(dv); + _DVFree(dvNext); return 0; } diff --git a/src/gb/extra/proxy.c b/src/gb/extra/proxy.c index 03119226e..5b5dbc3bd 100644 --- a/src/gb/extra/proxy.c +++ b/src/gb/extra/proxy.c @@ -64,7 +64,7 @@ static void _init(struct GBVideoProxyRenderer* proxyRenderer) { } } -static void _reset(struct GBVideoProxyRenderer* proxyRenderer, enum GBModel model) { +static void _reset(struct GBVideoProxyRenderer* proxyRenderer) { memcpy(proxyRenderer->logger->oam, &proxyRenderer->d.oam->raw, GB_SIZE_OAM); memcpy(proxyRenderer->logger->vram, proxyRenderer->d.vram, GB_SIZE_VRAM); @@ -83,7 +83,7 @@ void GBVideoProxyRendererShim(struct GBVideo* video, struct GBVideoProxyRenderer renderer->d.vram = video->vram; renderer->d.oam = &video->oam; _init(renderer); - _reset(renderer, video->p->model); + _reset(renderer); } void GBVideoProxyRendererUnshim(struct GBVideo* video, struct GBVideoProxyRenderer* renderer) { diff --git a/src/gba/core.c b/src/gba/core.c index 81c9a29d7..8f3e64a57 100644 --- a/src/gba/core.c +++ b/src/gba/core.c @@ -354,6 +354,7 @@ static void _GBACoreDesiredVideoDimensions(struct mCore* core, unsigned* width, struct GBACore* gbacore = (struct GBACore*) core; int scale = gbacore->glRenderer.scale; #else + UNUSED(core); int scale = 1; #endif @@ -507,7 +508,7 @@ static void _GBACoreReset(struct mCore* core) { || gbacore->glRenderer.outputTex != (unsigned) -1 #endif ) { - struct GBAVideoRenderer* renderer; + struct GBAVideoRenderer* renderer = NULL; if (gbacore->renderer.outputBuffer) { renderer = &gbacore->renderer.d; } @@ -526,13 +527,15 @@ static void _GBACoreReset(struct mCore* core) { } #endif #ifndef MINIMAL_CORE - if (core->videoLogger) { + if (renderer && core->videoLogger) { gbacore->proxyRenderer.logger = core->videoLogger; GBAVideoProxyRendererCreate(&gbacore->proxyRenderer, renderer); renderer = &gbacore->proxyRenderer.d; } #endif - GBAVideoAssociateRenderer(&gba->video, renderer); + if (renderer) { + GBAVideoAssociateRenderer(&gba->video, renderer); + } } #ifndef MINIMAL_CORE @@ -584,7 +587,7 @@ static void _GBACoreReset(struct mCore* core) { bios = NULL; } } - if (bios) { + if (found && bios) { GBALoadBIOS(gba, bios); } } diff --git a/src/gba/renderers/gl.c b/src/gba/renderers/gl.c index f9f2c4f33..7bc564648 100644 --- a/src/gba/renderers/gl.c +++ b/src/gba/renderers/gl.c @@ -1161,7 +1161,6 @@ void _cleanRegister(struct GBAVideoGLRenderer* glRenderer, int address, uint16_t break; case REG_BLDCNT: GBAVideoGLRendererWriteBLDCNT(glRenderer, value); - value &= 0x3FFF; break; case REG_BLDALPHA: glRenderer->blda = value & 0x1F; @@ -1172,7 +1171,6 @@ void _cleanRegister(struct GBAVideoGLRenderer* glRenderer, int address, uint16_t if (glRenderer->bldb > 0x10) { glRenderer->bldb = 0x10; } - value &= 0x1F1F; break; case REG_BLDY: glRenderer->bldy = value; @@ -1505,7 +1503,10 @@ void GBAVideoGLRendererGetPixels(struct GBAVideoRenderer* renderer, size_t* stri } void GBAVideoGLRendererPutPixels(struct GBAVideoRenderer* renderer, size_t stride, const void* pixels) { - + // TODO + UNUSED(renderer); + UNUSED(stride); + UNUSED(pixels); } static void _enableBg(struct GBAVideoGLRenderer* renderer, int bg, bool active) { diff --git a/src/sm83/debugger/cli-debugger.c b/src/sm83/debugger/cli-debugger.c index 86022db5e..c227d845e 100644 --- a/src/sm83/debugger/cli-debugger.c +++ b/src/sm83/debugger/cli-debugger.c @@ -46,7 +46,7 @@ static void _disassemble(struct CLIDebuggerSystem* debugger, struct CLIDebugVect size = 1; } else { size = dv->intValue; - dv = dv->next; // TODO: Check for excess args + // TODO: Check for excess args } size_t i; @@ -57,7 +57,7 @@ static void _disassemble(struct CLIDebuggerSystem* debugger, struct CLIDebugVect static inline uint16_t _printLine(struct CLIDebugger* debugger, uint16_t address, int segment) { struct CLIDebuggerBackend* be = debugger->backend; - struct SM83InstructionInfo info = {0}; + struct SM83InstructionInfo info = {{0}, 0}; char disassembly[48]; char* disPtr = disassembly; if (segment >= 0) { diff --git a/src/util/gui.c b/src/util/gui.c index 8d5a1ec59..82165939a 100644 --- a/src/util/gui.c +++ b/src/util/gui.c @@ -16,7 +16,8 @@ void GUIInit(struct GUIParams* params) { void GUIPollInput(struct GUIParams* params, uint32_t* newInputOut, uint32_t* heldInput) { uint32_t input = params->pollInput(¶ms->keyMap); uint32_t newInput = 0; - for (int i = 0; i < GUI_INPUT_MAX; ++i) { + int i; + for (i = 0; i < GUI_INPUT_MAX; ++i) { if (input & (1 << i)) { ++params->inputHistory[i]; } else { @@ -66,7 +67,8 @@ enum GUICursorState GUIPollCursor(struct GUIParams* params, unsigned* x, unsigne } void GUIInvalidateKeys(struct GUIParams* params) { - for (int i = 0; i < GUI_INPUT_MAX; ++i) { + int i; + for (i = 0; i < GUI_INPUT_MAX; ++i) { params->inputHistory[i] = 0; } } diff --git a/src/util/hash.c b/src/util/hash.c index 9db508c13..c75cb2ab1 100644 --- a/src/util/hash.c +++ b/src/util/hash.c @@ -60,7 +60,8 @@ uint32_t hash32(const void* key, int len, uint32_t seed) { const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); - for(int i = -nblocks; i; i++) + int i; + for(i = -nblocks; i; i++) { uint32_t k1 = getblock32(blocks,i); @@ -82,10 +83,15 @@ uint32_t hash32(const void* key, int len, uint32_t seed) { switch(len & 3) { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; - case 1: k1 ^= tail[0]; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + case 3: + k1 ^= tail[2] << 16; + // Fall through + case 2: + k1 ^= tail[1] << 8; + // Fall through + case 1: + k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; }; //---------- diff --git a/src/util/string.c b/src/util/string.c index 352daa91b..46e52fba7 100644 --- a/src/util/string.c +++ b/src/util/string.c @@ -234,8 +234,8 @@ char* utf16to8(const uint16_t* utf16, size_t length) { extern const uint16_t gbkUnicodeTable[]; char* gbkToUtf8(const char* gbk, size_t length) { - char* utf8 = 0; - char* utf8Offset; + char* utf8 = NULL; + char* utf8Offset = NULL; size_t offset; uint8_t gbk1 = 0; char buffer[4]; @@ -266,17 +266,17 @@ char* gbkToUtf8(const char* gbk, size_t length) { size_t bytes = toUtf8(unichar, buffer); utf8Length += bytes; - if (utf8Length < utf8TotalBytes) { - memcpy(utf8Offset, buffer, bytes); - utf8Offset += bytes; - } else if (!utf8) { + if (!utf8) { utf8 = malloc(length); if (!utf8) { - return 0; + return NULL; } utf8TotalBytes = length; memcpy(utf8, buffer, bytes); utf8Offset = utf8 + bytes; + } else if (utf8Length < utf8TotalBytes) { + memcpy(utf8Offset, buffer, bytes); + utf8Offset += bytes; } else if (utf8Length >= utf8TotalBytes) { ptrdiff_t o = utf8Offset - utf8; char* newUTF8 = realloc(utf8, utf8TotalBytes * 2); diff --git a/src/util/vfs.c b/src/util/vfs.c index 82035ec1e..77300def6 100644 --- a/src/util/vfs.c +++ b/src/util/vfs.c @@ -199,7 +199,7 @@ void separatePath(const char* path, char* dirname, char* basename, char* extensi if (PATH_MAX <= len) { len = PATH_MAX - 1; } - strncpy(extension, dotPoint, len); + strncpy(extension, dotPoint, PATH_MAX - 1); extension[len] = '\0'; } else { extension[0] = '\0'; From b2d406a411b31acce5bbf0246af32a80c22ca834 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 16 Jan 2020 20:10:20 -0800 Subject: [PATCH 14/20] Third-Party: Update LZMA SDK --- src/third-party/lzma/7z.h | 12 +- src/third-party/lzma/7zAlloc.c | 12 +- src/third-party/lzma/7zAlloc.h | 20 +- src/third-party/lzma/7zArcIn.c | 135 +- src/third-party/lzma/7zBuf.c | 12 +- src/third-party/lzma/7zBuf.h | 10 +- src/third-party/lzma/7zBuf2.c | 17 +- src/third-party/lzma/7zCrc.c | 16 +- src/third-party/lzma/7zCrcOpt.c | 50 +- src/third-party/lzma/7zDec.c | 64 +- src/third-party/lzma/7zFile.c | 26 +- src/third-party/lzma/7zFile.h | 8 +- src/third-party/lzma/7zStream.c | 111 +- src/third-party/lzma/7zTypes.h | 227 +- src/third-party/lzma/7zVersion.h | 22 +- src/third-party/lzma/Aes.c | 67 +- src/third-party/lzma/AesOpt.c | 4 +- src/third-party/lzma/Alloc.c | 399 ++- src/third-party/lzma/Alloc.h | 20 +- src/third-party/lzma/Bcj2.c | 13 +- src/third-party/lzma/Bcj2Enc.c | 11 +- src/third-party/lzma/Bra.c | 289 +- src/third-party/lzma/Bra86.c | 4 +- src/third-party/lzma/BraIA64.c | 82 +- src/third-party/lzma/Compiler.h | 3 +- src/third-party/lzma/CpuArch.c | 39 +- src/third-party/lzma/CpuArch.h | 182 +- src/third-party/lzma/DllSecur.c | 108 + src/third-party/lzma/DllSecur.h | 20 + src/third-party/lzma/LzFind.c | 321 +- src/third-party/lzma/LzFind.h | 12 +- src/third-party/lzma/LzFindMt.c | 182 +- src/third-party/lzma/LzFindMt.h | 18 +- src/third-party/lzma/Lzma2Dec.c | 354 ++- src/third-party/lzma/Lzma2Dec.h | 66 +- src/third-party/lzma/Lzma2DecMt.c | 1082 +++++++ src/third-party/lzma/Lzma2DecMt.h | 79 + src/third-party/lzma/Lzma2Enc.c | 695 +++-- src/third-party/lzma/Lzma2Enc.h | 45 +- src/third-party/lzma/Lzma86Dec.c | 6 +- src/third-party/lzma/Lzma86Enc.c | 12 +- src/third-party/lzma/LzmaDec.c | 439 +-- src/third-party/lzma/LzmaDec.h | 47 +- src/third-party/lzma/LzmaEnc.c | 2726 ++++++++++------- src/third-party/lzma/LzmaEnc.h | 48 +- src/third-party/lzma/MtCoder.c | 744 +++-- src/third-party/lzma/MtCoder.h | 153 +- src/third-party/lzma/MtDec.c | 1138 +++++++ src/third-party/lzma/MtDec.h | 201 ++ src/third-party/lzma/Ppmd.h | 6 +- src/third-party/lzma/Ppmd7.c | 34 +- src/third-party/lzma/Ppmd7.h | 30 +- src/third-party/lzma/Ppmd7Dec.c | 36 +- src/third-party/lzma/Ppmd7Enc.c | 4 +- src/third-party/lzma/Sha256.c | 6 +- src/third-party/lzma/Threads.c | 14 +- src/third-party/lzma/Threads.h | 5 +- src/third-party/lzma/Util/7z/7zMain.c | 176 +- src/third-party/lzma/Util/Lzma/LzmaUtil.c | 42 +- src/third-party/lzma/Util/Lzma/LzmaUtil.dsp | 8 +- src/third-party/lzma/Util/SfxSetup/SfxSetup.c | 51 +- .../lzma/Util/SfxSetup/SfxSetup.dsp | 8 + src/third-party/lzma/Util/SfxSetup/makefile | 2 + .../lzma/Util/SfxSetup/makefile_con | 2 + src/third-party/lzma/Xz.c | 18 +- src/third-party/lzma/Xz.h | 239 +- src/third-party/lzma/XzCrc64.c | 22 +- src/third-party/lzma/XzCrc64Opt.c | 30 +- src/third-party/lzma/XzDec.c | 2265 ++++++++++++-- src/third-party/lzma/XzEnc.c | 1355 ++++++-- src/third-party/lzma/XzEnc.h | 27 +- src/third-party/lzma/XzIn.c | 62 +- src/util/vfs/vfs-lzma.c | 18 +- 73 files changed, 11359 insertions(+), 3452 deletions(-) create mode 100644 src/third-party/lzma/DllSecur.c create mode 100644 src/third-party/lzma/DllSecur.h create mode 100644 src/third-party/lzma/Lzma2DecMt.c create mode 100644 src/third-party/lzma/Lzma2DecMt.h create mode 100644 src/third-party/lzma/MtDec.c create mode 100644 src/third-party/lzma/MtDec.h diff --git a/src/third-party/lzma/7z.h b/src/third-party/lzma/7z.h index 216f381ca..82813c298 100644 --- a/src/third-party/lzma/7z.h +++ b/src/third-party/lzma/7z.h @@ -1,5 +1,5 @@ /* 7z.h -- 7z interface -2015-11-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_H #define __7Z_H @@ -98,7 +98,7 @@ UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *stream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain); + ISzAllocPtr allocMain); typedef struct { @@ -131,7 +131,7 @@ typedef struct #define SzArEx_GetFileSize(p, i) ((p)->UnpackPositions[(i) + 1] - (p)->UnpackPositions[i]) void SzArEx_Init(CSzArEx *p); -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc); +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc); UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize); @@ -179,8 +179,8 @@ SRes SzArEx_Extract( size_t *outBufferSize, /* buffer size for output buffer */ size_t *offset, /* offset of stream for required file in *outBuffer */ size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp); + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp); /* @@ -195,7 +195,7 @@ SZ_ERROR_FAIL */ SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp); + ISzAllocPtr allocMain, ISzAllocPtr allocTemp); EXTERN_C_END diff --git a/src/third-party/lzma/7zAlloc.c b/src/third-party/lzma/7zAlloc.c index 360da0516..ea32809c6 100644 --- a/src/third-party/lzma/7zAlloc.c +++ b/src/third-party/lzma/7zAlloc.c @@ -1,8 +1,10 @@ /* 7zAlloc.c -- Allocation functions -2015-11-09 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #include "7zAlloc.h" /* #define _SZ_ALLOC_DEBUG */ @@ -20,7 +22,7 @@ int g_allocCountTemp = 0; #endif -void *SzAlloc(void *p, size_t size) +void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0) @@ -32,7 +34,7 @@ void *SzAlloc(void *p, size_t size) return malloc(size); } -void SzFree(void *p, void *address) +void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG @@ -45,7 +47,7 @@ void SzFree(void *p, void *address) free(address); } -void *SzAllocTemp(void *p, size_t size) +void *SzAllocTemp(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0) @@ -60,7 +62,7 @@ void *SzAllocTemp(void *p, size_t size) return malloc(size); } -void SzFreeTemp(void *p, void *address) +void SzFreeTemp(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG diff --git a/src/third-party/lzma/7zAlloc.h b/src/third-party/lzma/7zAlloc.h index 4d7502ffb..c0f89d73c 100644 --- a/src/third-party/lzma/7zAlloc.h +++ b/src/third-party/lzma/7zAlloc.h @@ -1,23 +1,19 @@ /* 7zAlloc.h -- Allocation functions -2013-03-25 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_ALLOC_H #define __7Z_ALLOC_H -#include +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN -void *SzAlloc(void *p, size_t size); -void SzFree(void *p, void *address); +void *SzAlloc(ISzAllocPtr p, size_t size); +void SzFree(ISzAllocPtr p, void *address); -void *SzAllocTemp(void *p, size_t size); -void SzFreeTemp(void *p, void *address); +void *SzAllocTemp(ISzAllocPtr p, size_t size); +void SzFreeTemp(ISzAllocPtr p, void *address); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif diff --git a/src/third-party/lzma/7zArcIn.c b/src/third-party/lzma/7zArcIn.c index 080d8df2a..68cc12ff4 100644 --- a/src/third-party/lzma/7zArcIn.c +++ b/src/third-party/lzma/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2015-11-18 : Igor Pavlov : Public domain */ +2018-12-31 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,7 +11,7 @@ #include "CpuArch.h" #define MY_ALLOC(T, p, size, alloc) { \ - if ((p = (T *)IAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } + if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } #define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } @@ -19,7 +19,7 @@ { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } #define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } #define k7zMajorVersion 0 @@ -60,7 +60,7 @@ const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; #define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAlloc *alloc) +static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) { if (num == 0) { @@ -75,18 +75,18 @@ static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAlloc *alloc) return SZ_OK; } -void SzBitUi32s_Free(CSzBitUi32s *p, ISzAlloc *alloc) +void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } #define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -void SzBitUi64s_Free(CSzBitUi64s *p, ISzAlloc *alloc) +void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } @@ -107,18 +107,18 @@ static void SzAr_Init(CSzAr *p) p->CodersData = NULL; } -static void SzAr_Free(CSzAr *p, ISzAlloc *alloc) +static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->PackPositions); + ISzAlloc_Free(alloc, p->PackPositions); SzBitUi32s_Free(&p->FolderCRCs, alloc); - IAlloc_Free(alloc, p->FoCodersOffsets); - IAlloc_Free(alloc, p->FoStartPackStreamIndex); - IAlloc_Free(alloc, p->FoToCoderUnpackSizes); - IAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); - IAlloc_Free(alloc, p->CoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoCodersOffsets); + ISzAlloc_Free(alloc, p->FoStartPackStreamIndex); + ISzAlloc_Free(alloc, p->FoToCoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); + ISzAlloc_Free(alloc, p->CoderUnpackSizes); - IAlloc_Free(alloc, p->CodersData); + ISzAlloc_Free(alloc, p->CodersData); SzAr_Init(p); } @@ -147,16 +147,16 @@ void SzArEx_Init(CSzArEx *p) SzBitUi64s_Init(&p->CTime); } -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc) +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->UnpackPositions); - IAlloc_Free(alloc, p->IsDirs); + ISzAlloc_Free(alloc, p->UnpackPositions); + ISzAlloc_Free(alloc, p->IsDirs); - IAlloc_Free(alloc, p->FolderToFile); - IAlloc_Free(alloc, p->FileToFolder); + ISzAlloc_Free(alloc, p->FolderToFile); + ISzAlloc_Free(alloc, p->FileToFolder); - IAlloc_Free(alloc, p->FileNameOffsets); - IAlloc_Free(alloc, p->FileNames); + ISzAlloc_Free(alloc, p->FileNameOffsets); + ISzAlloc_Free(alloc, p->FileNames); SzBitUi32s_Free(&p->CRCs, alloc); SzBitUi32s_Free(&p->Attribs, alloc); @@ -305,7 +305,7 @@ static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) return sum; } -static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) { Byte allAreDefined; Byte *v2; @@ -328,12 +328,12 @@ static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, I { unsigned numBits = (unsigned)numItems & 7; if (numBits != 0) - v2[numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); + v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); } return SZ_OK; } -static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { UInt32 i; CSzData sd; @@ -354,7 +354,7 @@ static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *c return SZ_OK; } -static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { SzBitUi32s_Free(crcs, alloc); RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)); @@ -380,7 +380,7 @@ static SRes SkipBitUi32s(CSzData *sd, UInt32 numItems) return SZ_OK; } -static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAlloc *alloc) +static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) { RINOK(SzReadNumber32(sd, &p->NumPackStreams)); @@ -635,7 +635,7 @@ static SRes ReadUnpackInfo(CSzAr *p, CSzData *sd2, UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd; @@ -666,7 +666,7 @@ static SRes ReadUnpackInfo(CSzAr *p, MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); startBufPtr = sd.Data; @@ -788,13 +788,9 @@ static SRes ReadUnpackInfo(CSzAr *p, numCodersOutStreams += numCoders; if (numCodersOutStreams < numCoders) return SZ_ERROR_UNSUPPORTED; - - packStreamIndex += numPackStreams; - if (packStreamIndex < numPackStreams) - return SZ_ERROR_UNSUPPORTED; - - if (packStreamIndex > p->NumPackStreams) + if (numPackStreams > p->NumPackStreams - packStreamIndex) return SZ_ERROR_ARCHIVE; + packStreamIndex += numPackStreams; } } @@ -938,7 +934,7 @@ static SRes SzReadStreamsInfo(CSzAr *p, UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, UInt64 *dataOffset, CSubStreamInfo *ssi, - ISzAlloc *alloc) + ISzAllocPtr alloc) { UInt64 type; @@ -980,7 +976,7 @@ static SRes SzReadAndDecodePackedStreams( UInt32 numFoldersMax, UInt64 baseOffset, CSzAr *p, - ISzAlloc *allocTemp) + ISzAllocPtr allocTemp) { UInt64 dataStartPos; UInt32 fo; @@ -1047,7 +1043,7 @@ static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, CSzData *sd2, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd; UInt32 i; @@ -1100,17 +1096,15 @@ static SRes SzReadHeader2( CSzData *sd, ILookInStream *inStream, CBuf *tempBufs, UInt32 *numTempBufs, - ISzAlloc *allocMain, - ISzAlloc *allocTemp + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp ) { - UInt64 type; - UInt32 numFiles = 0; - UInt32 numEmptyStreams = 0; CSubStreamInfo ssi; - const Byte *emptyStreams = NULL; - const Byte *emptyFiles = NULL; +{ + UInt64 type; + SzData_Clear(&ssi.sdSizes); SzData_Clear(&ssi.sdCRCs); SzData_Clear(&ssi.sdNumSubStreams); @@ -1124,9 +1118,9 @@ static SRes SzReadHeader2( { for (;;) { - UInt64 type; - RINOK(ReadID(sd, &type)); - if (type == k7zIdEnd) + UInt64 type2; + RINOK(ReadID(sd, &type2)); + if (type2 == k7zIdEnd) break; RINOK(SkipData(sd)); } @@ -1164,6 +1158,13 @@ static SRes SzReadHeader2( if (type != k7zIdFilesInfo) return SZ_ERROR_ARCHIVE; +} + +{ + UInt32 numFiles = 0; + UInt32 numEmptyStreams = 0; + const Byte *emptyStreams = NULL; + const Byte *emptyFiles = NULL; RINOK(SzReadNumber32(sd, &numFiles)); p->NumFiles = numFiles; @@ -1462,7 +1463,7 @@ static SRes SzReadHeader2( if (ssi.sdNumSubStreams.Data && ssi.sdNumSubStreams.Size != 0) return SZ_ERROR_ARCHIVE; } - +} return SZ_OK; } @@ -1471,8 +1472,8 @@ static SRes SzReadHeader( CSzArEx *p, CSzData *sd, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 i; UInt32 numTempBufs = 0; @@ -1500,8 +1501,8 @@ static SRes SzReadHeader( static SRes SzArEx_Open2( CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { Byte header[k7zStartHeaderSize]; Int64 startArcPos; @@ -1512,7 +1513,7 @@ static SRes SzArEx_Open2( SRes res; startArcPos = 0; - RINOK(inStream->Seek(inStream, &startArcPos, SZ_SEEK_CUR)); + RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); @@ -1541,7 +1542,7 @@ static SRes SzArEx_Open2( { Int64 pos = 0; - RINOK(inStream->Seek(inStream, &pos, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); if ((UInt64)pos < startArcPos + nextHeaderOffset || (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) @@ -1622,7 +1623,7 @@ static SRes SzArEx_Open2( SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp) + ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); if (res != SZ_OK) @@ -1640,8 +1641,8 @@ SRes SzArEx_Extract( size_t *outBufferSize, size_t *offset, size_t *outSizeProcessed, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 folderIndex = p->FileToFolder[fileIndex]; SRes res = SZ_OK; @@ -1651,7 +1652,7 @@ SRes SzArEx_Extract( if (folderIndex == (UInt32)-1) { - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *blockIndex = folderIndex; *tempBuf = NULL; *outBufferSize = 0; @@ -1663,7 +1664,7 @@ SRes SzArEx_Extract( UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); /* UInt64 unpackSizeSpec = - p->UnpackPositions[p->FolderToFile[folderIndex + 1]] - + p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - p->UnpackPositions[p->FolderToFile[folderIndex]]; */ size_t unpackSize = (size_t)unpackSizeSpec; @@ -1671,7 +1672,7 @@ SRes SzArEx_Extract( if (unpackSize != unpackSizeSpec) return SZ_ERROR_MEM; *blockIndex = folderIndex; - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *tempBuf = NULL; if (res == SZ_OK) @@ -1679,7 +1680,7 @@ SRes SzArEx_Extract( *outBufferSize = unpackSize; if (unpackSize != 0) { - *tempBuf = (Byte *)IAlloc_Alloc(allocMain, unpackSize); + *tempBuf = (Byte *)ISzAlloc_Alloc(allocMain, unpackSize); if (*tempBuf == NULL) res = SZ_ERROR_MEM; } @@ -1696,7 +1697,7 @@ SRes SzArEx_Extract( { UInt64 unpackPos = p->UnpackPositions[fileIndex]; *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); - *outSizeProcessed = (size_t)(p->UnpackPositions[fileIndex + 1] - unpackPos); + *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); if (*offset + *outSizeProcessed > *outBufferSize) return SZ_ERROR_FAIL; if (SzBitWithVals_Check(&p->CRCs, fileIndex)) @@ -1743,7 +1744,7 @@ size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) { - Bool needSlash; + BoolInt needSlash; if (!p->FileNameOffsets) { *(--dest) = 0; diff --git a/src/third-party/lzma/7zBuf.c b/src/third-party/lzma/7zBuf.c index b0ac1104b..438bba68b 100644 --- a/src/third-party/lzma/7zBuf.c +++ b/src/third-party/lzma/7zBuf.c @@ -1,5 +1,5 @@ /* 7zBuf.c -- Byte Buffer -2013-01-21 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,7 +11,7 @@ void Buf_Init(CBuf *p) p->size = 0; } -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc) { p->size = 0; if (size == 0) @@ -19,8 +19,8 @@ int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) p->data = 0; return 1; } - p->data = (Byte *)alloc->Alloc(alloc, size); - if (p->data != 0) + p->data = (Byte *)ISzAlloc_Alloc(alloc, size); + if (p->data) { p->size = size; return 1; @@ -28,9 +28,9 @@ int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) return 0; } -void Buf_Free(CBuf *p, ISzAlloc *alloc) +void Buf_Free(CBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; } diff --git a/src/third-party/lzma/7zBuf.h b/src/third-party/lzma/7zBuf.h index e5f9218be..5942d6e62 100644 --- a/src/third-party/lzma/7zBuf.h +++ b/src/third-party/lzma/7zBuf.h @@ -1,5 +1,5 @@ /* 7zBuf.h -- Byte Buffer -2013-01-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_BUF_H #define __7Z_BUF_H @@ -15,8 +15,8 @@ typedef struct } CBuf; void Buf_Init(CBuf *p); -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc); -void Buf_Free(CBuf *p, ISzAlloc *alloc); +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc); +void Buf_Free(CBuf *p, ISzAllocPtr alloc); typedef struct { @@ -27,8 +27,8 @@ typedef struct void DynBuf_Construct(CDynBuf *p); void DynBuf_SeekToBeg(CDynBuf *p); -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc); -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc); +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc); +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc); EXTERN_C_END diff --git a/src/third-party/lzma/7zBuf2.c b/src/third-party/lzma/7zBuf2.c index 8e3fe9297..49b4343b6 100644 --- a/src/third-party/lzma/7zBuf2.c +++ b/src/third-party/lzma/7zBuf2.c @@ -1,5 +1,5 @@ /* 7zBuf2.c -- Byte Buffer -2014-08-22 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -19,19 +19,20 @@ void DynBuf_SeekToBeg(CDynBuf *p) p->pos = 0; } -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { if (size > p->size - p->pos) { size_t newSize = p->pos + size; Byte *data; newSize += newSize / 4; - data = (Byte *)alloc->Alloc(alloc, newSize); - if (data == 0) + data = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!data) return 0; p->size = newSize; - memcpy(data, p->data, p->pos); - alloc->Free(alloc, p->data); + if (p->pos != 0) + memcpy(data, p->data, p->pos); + ISzAlloc_Free(alloc, p->data); p->data = data; } if (size != 0) @@ -42,9 +43,9 @@ int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) return 1; } -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc) +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; p->pos = 0; diff --git a/src/third-party/lzma/7zCrc.c b/src/third-party/lzma/7zCrc.c index 607db3404..40ab75952 100644 --- a/src/third-party/lzma/7zCrc.c +++ b/src/third-party/lzma/7zCrc.c @@ -1,5 +1,5 @@ /* 7zCrc.c -- CRC32 init -2015-03-10 : Igor Pavlov : Public domain */ +2017-06-06 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -61,12 +61,12 @@ void MY_FAST_CALL CrcGenerateTable() UInt32 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); g_CrcTable[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC_NUM_TABLES; i++) { - UInt32 r = g_CrcTable[i - 256]; + UInt32 r = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); } @@ -86,8 +86,8 @@ void MY_FAST_CALL CrcGenerateTable() #ifdef MY_CPU_X86_OR_AMD64 if (!CPU_Is_InOrder()) - g_CrcUpdate = CrcUpdateT8; #endif + g_CrcUpdate = CrcUpdateT8; #endif #else @@ -101,7 +101,7 @@ void MY_FAST_CALL CrcGenerateTable() g_CrcUpdate = CrcUpdateT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT8; - // g_CrcUpdate = CrcUpdateT8; + g_CrcUpdate = CrcUpdateT8; #endif } else if (p[0] != 1 || p[1] != 2) @@ -111,14 +111,14 @@ void MY_FAST_CALL CrcGenerateTable() { for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) { - UInt32 x = g_CrcTable[i - 256]; + UInt32 x = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = CRC_UINT32_SWAP(x); } g_CrcUpdateT4 = CrcUpdateT1_BeT4; g_CrcUpdate = CrcUpdateT1_BeT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT1_BeT8; - // g_CrcUpdate = CrcUpdateT1_BeT8; + g_CrcUpdate = CrcUpdateT1_BeT8; #endif } } diff --git a/src/third-party/lzma/7zCrcOpt.c b/src/third-party/lzma/7zCrcOpt.c index 58628efe5..2ee0de845 100644 --- a/src/third-party/lzma/7zCrcOpt.c +++ b/src/third-party/lzma/7zCrcOpt.c @@ -1,5 +1,5 @@ /* 7zCrcOpt.c -- CRC32 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -18,10 +18,10 @@ UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const U { v ^= *(const UInt32 *)p; v = - table[0x300 + ((v ) & 0xFF)] - ^ table[0x200 + ((v >> 8) & 0xFF)] - ^ table[0x100 + ((v >> 16) & 0xFF)] - ^ table[0x000 + ((v >> 24))]; + (table + 0x300)[((v ) & 0xFF)] + ^ (table + 0x200)[((v >> 8) & 0xFF)] + ^ (table + 0x100)[((v >> 16) & 0xFF)] + ^ (table + 0x000)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); @@ -38,16 +38,16 @@ UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const U UInt32 d; v ^= *(const UInt32 *)p; v = - table[0x700 + ((v ) & 0xFF)] - ^ table[0x600 + ((v >> 8) & 0xFF)] - ^ table[0x500 + ((v >> 16) & 0xFF)] - ^ table[0x400 + ((v >> 24))]; + (table + 0x700)[((v ) & 0xFF)] + ^ (table + 0x600)[((v >> 8) & 0xFF)] + ^ (table + 0x500)[((v >> 16) & 0xFF)] + ^ (table + 0x400)[((v >> 24))]; d = *((const UInt32 *)p + 1); v ^= - table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); @@ -74,10 +74,10 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, co { v ^= *(const UInt32 *)p; v = - table[0x000 + ((v ) & 0xFF)] - ^ table[0x100 + ((v >> 8) & 0xFF)] - ^ table[0x200 + ((v >> 16) & 0xFF)] - ^ table[0x300 + ((v >> 24))]; + (table + 0x000)[((v ) & 0xFF)] + ^ (table + 0x100)[((v >> 8) & 0xFF)] + ^ (table + 0x200)[((v >> 16) & 0xFF)] + ^ (table + 0x300)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); @@ -96,16 +96,16 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, co UInt32 d; v ^= *(const UInt32 *)p; v = - table[0x400 + ((v ) & 0xFF)] - ^ table[0x500 + ((v >> 8) & 0xFF)] - ^ table[0x600 + ((v >> 16) & 0xFF)] - ^ table[0x700 + ((v >> 24))]; + (table + 0x400)[((v ) & 0xFF)] + ^ (table + 0x500)[((v >> 8) & 0xFF)] + ^ (table + 0x600)[((v >> 16) & 0xFF)] + ^ (table + 0x700)[((v >> 24))]; d = *((const UInt32 *)p + 1); v ^= - table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); diff --git a/src/third-party/lzma/7zDec.c b/src/third-party/lzma/7zDec.c index e39b4ff12..2a7b09030 100644 --- a/src/third-party/lzma/7zDec.c +++ b/src/third-party/lzma/7zDec.c @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2015-11-18 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -39,28 +39,28 @@ typedef struct { - IByteIn p; + IByteIn vt; const Byte *cur; const Byte *end; const Byte *begin; UInt64 processed; - Bool extra; + BoolInt extra; SRes res; - ILookInStream *inStream; + const ILookInStream *inStream; } CByteInToLook; -static Byte ReadByte(void *pp) +static Byte ReadByte(const IByteIn *pp) { - CByteInToLook *p = (CByteInToLook *)pp; + CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); if (p->cur != p->end) return *p->cur++; if (p->res == SZ_OK) { size_t size = p->cur - p->begin; p->processed += size; - p->res = p->inStream->Skip(p->inStream, size); + p->res = ILookInStream_Skip(p->inStream, size); size = (1 << 25); - p->res = p->inStream->Look(p->inStream, (const void **)&p->begin, &size); + p->res = ILookInStream_Look(p->inStream, (const void **)&p->begin, &size); p->cur = p->begin; p->end = p->begin + size; if (size != 0) @@ -70,14 +70,14 @@ static Byte ReadByte(void *pp) return 0; } -static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) +static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CPpmd7 ppmd; CByteInToLook s; SRes res = SZ_OK; - s.p.Read = ReadByte; + s.vt.Read = ReadByte; s.inStream = inStream; s.begin = s.end = s.cur = NULL; s.extra = False; @@ -103,7 +103,7 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I { CPpmd7z_RangeDec rc; Ppmd7z_RangeDec_CreateVTable(&rc); - rc.Stream = &s.p; + rc.Stream = &s.vt; if (!Ppmd7z_RangeDec_Init(&rc)) res = SZ_ERROR_DATA; else if (s.extra) @@ -113,7 +113,7 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I SizeT i; for (i = 0; i < outSize; i++) { - int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.p); + int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt); if (s.extra || sym < 0) break; outBuffer[i] = (Byte)sym; @@ -132,7 +132,7 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzmaDec state; SRes res = SZ_OK; @@ -149,14 +149,14 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -178,7 +178,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; } @@ -192,7 +192,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I #ifndef _7Z_NO_METHOD_LZMA2 static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzma2Dec state; SRes res = SZ_OK; @@ -211,14 +211,14 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -237,7 +237,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; } @@ -258,18 +258,18 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer size_t curSize = (1 << 18); if (curSize > inSize) curSize = (size_t)inSize; - RINOK(inStream->Look(inStream, &inBuf, &curSize)); + RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); if (curSize == 0) return SZ_ERROR_INPUT_EOF; memcpy(outBuffer, inBuf, curSize); outBuffer += curSize; inSize -= curSize; - RINOK(inStream->Skip((void *)inStream, curSize)); + RINOK(ILookInStream_Skip(inStream, curSize)); } return SZ_OK; } -static Bool IS_MAIN_METHOD(UInt32 m) +static BoolInt IS_MAIN_METHOD(UInt32 m) { switch (m) { @@ -286,7 +286,7 @@ static Bool IS_MAIN_METHOD(UInt32 m) return False; } -static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c) +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) { return c->NumStreams == 1 @@ -372,7 +372,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, const UInt64 *unpackSizes, const UInt64 *packPositions, ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, Byte *tempBuf[]) { UInt32 ci; @@ -404,7 +404,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, outSizeCur = (SizeT)unpackSize; if (outSizeCur != unpackSize) return SZ_ERROR_MEM; - temp = (Byte *)IAlloc_Alloc(allocMain, outSizeCur); + temp = (Byte *)ISzAlloc_Alloc(allocMain, outSizeCur); if (!temp && outSizeCur != 0) return SZ_ERROR_MEM; outBufCur = tempBuf[1 - ci] = temp; @@ -421,7 +421,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, return SZ_ERROR_UNSUPPORTED; } offset = packPositions[si]; - inSize = packPositions[si + 1] - offset; + inSize = packPositions[(size_t)si + 1] - offset; RINOK(LookInStream_SeekTo(inStream, startPos + offset)); if (coder->MethodID == k_Copy) @@ -460,7 +460,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, tempSizes[2] = (SizeT)s3Size; if (tempSizes[2] != s3Size) return SZ_ERROR_MEM; - tempBuf[2] = (Byte *)IAlloc_Alloc(allocMain, tempSizes[2]); + tempBuf[2] = (Byte *)ISzAlloc_Alloc(allocMain, tempSizes[2]); if (!tempBuf[2] && tempSizes[2] != 0) return SZ_ERROR_MEM; @@ -549,7 +549,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *inStream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain) + ISzAllocPtr allocMain) { SRes res; CSzFolder folder; @@ -557,7 +557,7 @@ SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, const Byte *data = p->CodersData + p->FoCodersOffsets[folderIndex]; sd.Data = data; - sd.Size = p->FoCodersOffsets[folderIndex + 1] - p->FoCodersOffsets[folderIndex]; + sd.Size = p->FoCodersOffsets[(size_t)folderIndex + 1] - p->FoCodersOffsets[folderIndex]; res = SzGetNextFolderItem(&folder, &sd); @@ -579,7 +579,7 @@ SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, outBuffer, (SizeT)outSize, allocMain, tempBuf); for (i = 0; i < 3; i++) - IAlloc_Free(allocMain, tempBuf[i]); + ISzAlloc_Free(allocMain, tempBuf[i]); if (res == SZ_OK) if (SzBitWithVals_Check(&p->FolderCRCs, folderIndex)) diff --git a/src/third-party/lzma/7zFile.c b/src/third-party/lzma/7zFile.c index 98fe7164f..e486901e3 100644 --- a/src/third-party/lzma/7zFile.c +++ b/src/third-party/lzma/7zFile.c @@ -1,5 +1,5 @@ /* 7zFile.c -- File IO -2009-11-24 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -238,49 +238,49 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) /* ---------- FileSeqInStream ---------- */ -static SRes FileSeqInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CFileSeqInStream *p = (CFileSeqInStream *)pp; + CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; } void FileSeqInStream_CreateVTable(CFileSeqInStream *p) { - p->s.Read = FileSeqInStream_Read; + p->vt.Read = FileSeqInStream_Read; } /* ---------- FileInStream ---------- */ -static SRes FileInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) { - CFileInStream *p = (CFileInStream *)pp; + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; } -static SRes FileInStream_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) { - CFileInStream *p = (CFileInStream *)pp; + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); return File_Seek(&p->file, pos, origin); } void FileInStream_CreateVTable(CFileInStream *p) { - p->s.Read = FileInStream_Read; - p->s.Seek = FileInStream_Seek; + p->vt.Read = FileInStream_Read; + p->vt.Seek = FileInStream_Seek; } /* ---------- FileOutStream ---------- */ -static size_t FileOutStream_Write(void *pp, const void *data, size_t size) +static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CFileOutStream *p = (CFileOutStream *)pp; + CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); File_Write(&p->file, data, &size); return size; } void FileOutStream_CreateVTable(CFileOutStream *p) { - p->s.Write = FileOutStream_Write; + p->vt.Write = FileOutStream_Write; } diff --git a/src/third-party/lzma/7zFile.h b/src/third-party/lzma/7zFile.h index d62a19260..7e263bea1 100644 --- a/src/third-party/lzma/7zFile.h +++ b/src/third-party/lzma/7zFile.h @@ -1,5 +1,5 @@ /* 7zFile.h -- File IO -2013-01-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_FILE_H #define __7Z_FILE_H @@ -54,7 +54,7 @@ WRes File_GetLength(CSzFile *p, UInt64 *length); typedef struct { - ISeqInStream s; + ISeqInStream vt; CSzFile file; } CFileSeqInStream; @@ -63,7 +63,7 @@ void FileSeqInStream_CreateVTable(CFileSeqInStream *p); typedef struct { - ISeekInStream s; + ISeekInStream vt; CSzFile file; } CFileInStream; @@ -72,7 +72,7 @@ void FileInStream_CreateVTable(CFileInStream *p); typedef struct { - ISeqOutStream s; + ISeqOutStream vt; CSzFile file; } CFileOutStream; diff --git a/src/third-party/lzma/7zStream.c b/src/third-party/lzma/7zStream.c index 5a92d532c..579741fad 100644 --- a/src/third-party/lzma/7zStream.c +++ b/src/third-party/lzma/7zStream.c @@ -1,5 +1,5 @@ /* 7zStream.c -- 7z Stream functions -2013-11-12 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,12 +7,12 @@ #include "7zTypes.h" -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType) +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -21,40 +21,42 @@ SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorT return SZ_OK; } -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size) +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) { return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf) +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) { size_t processed = 1; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; } -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset) + + +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) { Int64 t = offset; - return stream->Seek(stream, &t, SZ_SEEK_SET); + return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); } -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size) +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) { const void *lookBuf; if (*size == 0) return SZ_OK; - RINOK(stream->Look(stream, &lookBuf, size)); + RINOK(ILookInStream_Look(stream, &lookBuf, size)); memcpy(buf, lookBuf, *size); - return stream->Skip(stream, *size); + return ILookInStream_Skip(stream, *size); } -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType) +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ILookInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -63,61 +65,67 @@ SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes erro return SZ_OK; } -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size) +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) { return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -static SRes LookToRead_Look_Lookahead(void *pp, const void **buf, size_t *size) + + +#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); + +static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - size2 = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, &size2); + p->size = 0; + size2 = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, &size2); p->size = size2; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Look_Exact(void *pp, const void **buf, size_t *size) +static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - if (*size > LookToRead_BUF_SIZE) - *size = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, size); + p->size = 0; + if (*size > p->bufSize) + *size = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, size); size2 = p->size = *size; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Skip(void *pp, size_t offset) +static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos += offset; return SZ_OK; } -static SRes LookToRead_Read(void *pp, void *buf, size_t *size) +static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t rem = p->size - p->pos; if (rem == 0) - return p->realStream->Read(p->realStream, buf, size); + return ISeekInStream_Read(p->realStream, buf, size); if (rem > *size) rem = *size; memcpy(buf, p->buf + p->pos, rem); @@ -126,46 +134,43 @@ static SRes LookToRead_Read(void *pp, void *buf, size_t *size) return SZ_OK; } -static SRes LookToRead_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos = p->size = 0; - return p->realStream->Seek(p->realStream, pos, origin); + return ISeekInStream_Seek(p->realStream, pos, origin); } -void LookToRead_CreateVTable(CLookToRead *p, int lookahead) +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) { - p->s.Look = lookahead ? - LookToRead_Look_Lookahead : - LookToRead_Look_Exact; - p->s.Skip = LookToRead_Skip; - p->s.Read = LookToRead_Read; - p->s.Seek = LookToRead_Seek; + p->vt.Look = lookahead ? + LookToRead2_Look_Lookahead : + LookToRead2_Look_Exact; + p->vt.Skip = LookToRead2_Skip; + p->vt.Read = LookToRead2_Read; + p->vt.Seek = LookToRead2_Seek; } -void LookToRead_Init(CLookToRead *p) -{ - p->pos = p->size = 0; -} -static SRes SecToLook_Read(void *pp, void *buf, size_t *size) + +static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToLook *p = (CSecToLook *)pp; + CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); return LookInStream_LookRead(p->realStream, buf, size); } void SecToLook_CreateVTable(CSecToLook *p) { - p->s.Read = SecToLook_Read; + p->vt.Read = SecToLook_Read; } -static SRes SecToRead_Read(void *pp, void *buf, size_t *size) +static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToRead *p = (CSecToRead *)pp; - return p->realStream->Read(p->realStream, buf, size); + CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); + return ILookInStream_Read(p->realStream, buf, size); } void SecToRead_CreateVTable(CSecToRead *p) { - p->s.Read = SecToRead_Read; + p->vt.Read = SecToRead_Read; } diff --git a/src/third-party/lzma/7zTypes.h b/src/third-party/lzma/7zTypes.h index 903047b10..593f5aa25 100644 --- a/src/third-party/lzma/7zTypes.h +++ b/src/third-party/lzma/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2013-11-12 : Igor Pavlov : Public domain */ +2018-08-04 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H @@ -42,13 +42,23 @@ EXTERN_C_BEGIN typedef int SRes; + #ifdef _WIN32 + /* typedef DWORD WRes; */ typedef unsigned WRes; +#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) + #else + typedef int WRes; +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_WIN32 +#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000))) + #endif + #ifndef RINOK #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } #endif @@ -93,7 +103,8 @@ typedef UInt32 SizeT; typedef size_t SizeT; #endif -typedef int Bool; +typedef int BoolInt; +/* typedef BoolInt Bool; */ #define True 1 #define False 0 @@ -112,48 +123,72 @@ typedef int Bool; #define MY_NO_INLINE #endif +#define MY_FORCE_INLINE __forceinline + #define MY_CDECL __cdecl #define MY_FAST_CALL __fastcall #else #define MY_NO_INLINE +#define MY_FORCE_INLINE #define MY_CDECL #define MY_FAST_CALL +/* inline keyword : for C++ / C99 */ + +/* GCC, clang: */ +/* +#if defined (__GNUC__) && (__GNUC__ >= 4) +#define MY_FORCE_INLINE __attribute__((always_inline)) +#define MY_NO_INLINE __attribute__((noinline)) +#endif +*/ + #endif /* The following interfaces use first parameter as pointer to structure */ -typedef struct +typedef struct IByteIn IByteIn; +struct IByteIn { - Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ -} IByteIn; + Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */ +}; +#define IByteIn_Read(p) (p)->Read(p) -typedef struct -{ - void (*Write)(void *p, Byte b); -} IByteOut; -typedef struct +typedef struct IByteOut IByteOut; +struct IByteOut { - SRes (*Read)(void *p, void *buf, size_t *size); + void (*Write)(const IByteOut *p, Byte b); +}; +#define IByteOut_Write(p, b) (p)->Write(p, b) + + +typedef struct ISeqInStream ISeqInStream; +struct ISeqInStream +{ + SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) < input(*size)) is allowed */ -} ISeqInStream; +}; +#define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size) /* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf); -typedef struct + +typedef struct ISeqOutStream ISeqOutStream; +struct ISeqOutStream { - size_t (*Write)(void *p, const void *buf, size_t size); + size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size); /* Returns: result - the number of actually written bytes. (result < size) means error */ -} ISeqOutStream; +}; +#define ISeqOutStream_Write(p, buf, size) (p)->Write(p, buf, size) typedef enum { @@ -162,78 +197,162 @@ typedef enum SZ_SEEK_END = 2 } ESzSeek; -typedef struct -{ - SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ISeekInStream; -typedef struct +typedef struct ISeekInStream ISeekInStream; +struct ISeekInStream { - SRes (*Look)(void *p, const void **buf, size_t *size); + SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin); +}; +#define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + + +typedef struct ILookInStream ILookInStream; +struct ILookInStream +{ + SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) > input(*size)) is not allowed (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(void *p, size_t offset); + SRes (*Skip)(const ILookInStream *p, size_t offset); /* offset must be <= output(*size) of Look */ - SRes (*Read)(void *p, void *buf, size_t *size); + SRes (*Read)(const ILookInStream *p, void *buf, size_t *size); /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ILookInStream; + SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin); +}; -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); +#define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size) +#define ILookInStream_Skip(p, offset) (p)->Skip(p, offset) +#define ILookInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + + +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset); /* reads via ILookInStream::Read */ -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size); + -#define LookToRead_BUF_SIZE (1 << 14) typedef struct { - ILookInStream s; - ISeekInStream *realStream; + ILookInStream vt; + const ISeekInStream *realStream; + size_t pos; - size_t size; - Byte buf[LookToRead_BUF_SIZE]; -} CLookToRead; + size_t size; /* it's data size */ + + /* the following variables must be set outside */ + Byte *buf; + size_t bufSize; +} CLookToRead2; + +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead); + +#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; } -void LookToRead_CreateVTable(CLookToRead *p, int lookahead); -void LookToRead_Init(CLookToRead *p); typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToLook; void SecToLook_CreateVTable(CSecToLook *p); + + typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToRead; void SecToRead_CreateVTable(CSecToRead *p); -typedef struct + +typedef struct ICompressProgress ICompressProgress; + +struct ICompressProgress { - SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); + SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize); /* Returns: result. (result != SZ_OK) means break. Value (UInt64)(Int64)-1 for size means unknown value. */ -} ICompressProgress; +}; +#define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize) -typedef struct + + +typedef struct ISzAlloc ISzAlloc; +typedef const ISzAlloc * ISzAllocPtr; + +struct ISzAlloc { - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; + void *(*Alloc)(ISzAllocPtr p, size_t size); + void (*Free)(ISzAllocPtr p, void *address); /* address can be 0 */ +}; + +#define ISzAlloc_Alloc(p, size) (p)->Alloc(p, size) +#define ISzAlloc_Free(p, a) (p)->Free(p, a) + +/* deprecated */ +#define IAlloc_Alloc(p, size) ISzAlloc_Alloc(p, size) +#define IAlloc_Free(p, a) ISzAlloc_Free(p, a) + + + + + +#ifndef MY_offsetof + #ifdef offsetof + #define MY_offsetof(type, m) offsetof(type, m) + /* + #define MY_offsetof(type, m) FIELD_OFFSET(type, m) + */ + #else + #define MY_offsetof(type, m) ((size_t)&(((type *)0)->m)) + #endif +#endif + + + +#ifndef MY_container_of + +/* +#define MY_container_of(ptr, type, m) container_of(ptr, type, m) +#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) +#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) +#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) +*/ + +/* + GCC shows warning: "perhaps the 'offsetof' macro was used incorrectly" + GCC 3.4.4 : classes with constructor + GCC 4.8.1 : classes with non-public variable members" +*/ + +#define MY_container_of(ptr, type, m) ((type *)((char *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) + + +#endif + +#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(ptr)) + +/* +#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +*/ +#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m) + +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +/* +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m) +*/ + -#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) -#define IAlloc_Free(p, a) (p)->Free((p), a) #ifdef _WIN32 diff --git a/src/third-party/lzma/7zVersion.h b/src/third-party/lzma/7zVersion.h index 52ad3c1b1..0074c64be 100644 --- a/src/third-party/lzma/7zVersion.h +++ b/src/third-party/lzma/7zVersion.h @@ -1,14 +1,21 @@ -#define MY_VER_MAJOR 15 -#define MY_VER_MINOR 14 +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "15.14" -#define MY_VERSION "15.14" -#define MY_DATE "2015-12-31" +#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2019-02-21" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2015 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR @@ -16,4 +23,5 @@ #define MY_COPYRIGHT MY_COPYRIGHT_PD #endif -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/src/third-party/lzma/Aes.c b/src/third-party/lzma/Aes.c index b0263c6cc..8f7d50ea2 100644 --- a/src/third-party/lzma/Aes.c +++ b/src/third-party/lzma/Aes.c @@ -1,5 +1,5 @@ /* Aes.c -- AES encryption / decryption -2015-02-23 : Igor Pavlov : Public domain */ +2017-01-24 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -49,7 +49,13 @@ static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0 #define gb0(x) ( (x) & 0xFF) #define gb1(x) (((x) >> ( 8)) & 0xFF) #define gb2(x) (((x) >> (16)) & 0xFF) -#define gb3(x) (((x) >> (24)) & 0xFF) +#define gb3(x) (((x) >> (24))) + +#define gb(n, x) gb ## n(x) + +#define TT(x) (T + (x << 8)) +#define DD(x) (D + (x << 8)) + void AesGenTables(void) { @@ -63,10 +69,10 @@ void AesGenTables(void) UInt32 a1 = Sbox[i]; UInt32 a2 = xtime(a1); UInt32 a3 = a2 ^ a1; - T[ i] = Ui32(a2, a1, a1, a3); - T[0x100 + i] = Ui32(a3, a2, a1, a1); - T[0x200 + i] = Ui32(a1, a3, a2, a1); - T[0x300 + i] = Ui32(a1, a1, a3, a2); + TT(0)[i] = Ui32(a2, a1, a1, a3); + TT(1)[i] = Ui32(a3, a2, a1, a1); + TT(2)[i] = Ui32(a1, a3, a2, a1); + TT(3)[i] = Ui32(a1, a1, a3, a2); } { UInt32 a1 = InvS[i]; @@ -77,10 +83,10 @@ void AesGenTables(void) UInt32 aB = a8 ^ a2 ^ a1; UInt32 aD = a8 ^ a4 ^ a1; UInt32 aE = a8 ^ a4 ^ a2; - D[ i] = Ui32(aE, a9, aD, aB); - D[0x100 + i] = Ui32(aB, aE, a9, aD); - D[0x200 + i] = Ui32(aD, aB, aE, a9); - D[0x300 + i] = Ui32(a9, aD, aB, aE); + DD(0)[i] = Ui32(aE, a9, aD, aB); + DD(1)[i] = Ui32(aB, aE, a9, aD); + DD(2)[i] = Ui32(aD, aB, aE, a9); + DD(3)[i] = Ui32(a9, aD, aB, aE); } } @@ -99,7 +105,7 @@ void AesGenTables(void) } -#define HT(i, x, s) (T + (x << 8))[gb ## x(s[(i + x) & 3])] +#define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])] #define HT4(m, i, s, p) m[i] = \ HT(i, 0, s) ^ \ @@ -113,11 +119,11 @@ void AesGenTables(void) HT4(m, 2, s, p); \ HT4(m, 3, s, p); \ -#define FT(i, x) Sbox[gb ## x(m[(i + x) & 3])] +#define FT(i, x) Sbox[gb(x, m[(i + x) & 3])] #define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i]; -#define HD(i, x, s) (D + (x << 8))[gb ## x(s[(i - x) & 3])] +#define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])] #define HD4(m, i, s, p) m[i] = \ HD(i, 0, s) ^ \ @@ -131,7 +137,7 @@ void AesGenTables(void) HD4(m, 2, s, p); \ HD4(m, 3, s, p); \ -#define FD(i, x) InvS[gb ## x(m[(i - x) & 3])] +#define FD(i, x) InvS[gb(x, m[(i - x) & 3])] #define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) @@ -147,7 +153,7 @@ void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) for (; i < wSize; i++) { - UInt32 t = w[i - 1]; + UInt32 t = w[(size_t)i - 1]; unsigned rem = i % keySize; if (rem == 0) t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); @@ -167,10 +173,10 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) { UInt32 r = w[i]; w[i] = - D[ Sbox[gb0(r)]] ^ - D[0x100 + Sbox[gb1(r)]] ^ - D[0x200 + Sbox[gb2(r)]] ^ - D[0x300 + Sbox[gb3(r)]]; + DD(0)[Sbox[gb0(r)]] ^ + DD(1)[Sbox[gb1(r)]] ^ + DD(2)[Sbox[gb2(r)]] ^ + DD(3)[Sbox[gb3(r)]]; } } @@ -276,20 +282,25 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) for (; numBlocks != 0; numBlocks--) { UInt32 temp[4]; - Byte buf[16]; - int i; + unsigned i; if (++p[0] == 0) p[1]++; Aes_Encode(p + 4, temp, p); - SetUi32(buf, temp[0]); - SetUi32(buf + 4, temp[1]); - SetUi32(buf + 8, temp[2]); - SetUi32(buf + 12, temp[3]); - - for (i = 0; i < 16; i++) - *data++ ^= buf[i]; + for (i = 0; i < 4; i++, data += 4) + { + UInt32 t = temp[i]; + + #ifdef MY_CPU_LE_UNALIGN + *((UInt32 *)data) ^= t; + #else + data[0] ^= (t & 0xFF); + data[1] ^= ((t >> 8) & 0xFF); + data[2] ^= ((t >> 16) & 0xFF); + data[3] ^= ((t >> 24)); + #endif + } } } diff --git a/src/third-party/lzma/AesOpt.c b/src/third-party/lzma/AesOpt.c index e5d4d2611..0e7f49a1b 100644 --- a/src/third-party/lzma/AesOpt.c +++ b/src/third-party/lzma/AesOpt.c @@ -1,12 +1,12 @@ /* AesOpt.c -- Intel's AES -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-08 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if _MSC_VER >= 1500 +#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) #define USE_INTEL_AES #endif #endif diff --git a/src/third-party/lzma/Alloc.c b/src/third-party/lzma/Alloc.c index 9f1d036af..30b499e5f 100644 --- a/src/third-party/lzma/Alloc.c +++ b/src/third-party/lzma/Alloc.c @@ -1,8 +1,10 @@ /* Alloc.c -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2018-04-27 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #ifdef _WIN32 #include #endif @@ -14,20 +16,127 @@ /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) - return 0; + return NULL; #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -37,10 +146,8 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); } @@ -49,20 +156,18 @@ void MyFree(void *address) void *MidAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + return NULL; + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif - if (address == 0) + PRINT_FREE("Free-Mid", g_allocCountMid, address); + + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } @@ -79,10 +184,10 @@ typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); void SetLargePageSize() { #ifdef _7ZIP_LARGE_PAGES - SIZE_T size = 0; + SIZE_T size; GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (largePageMinimum == 0) + if (!largePageMinimum) return; size = largePageMinimum(); if (size == 0 || (size & (size - 1)) != 0) @@ -95,31 +200,36 @@ void SetLargePageSize() void *BigAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); - #endif + return NULL; + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES - if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) { - void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), - MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res != 0) - return res; + SIZE_T ps = g_LargePageSize; + if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) + { + size_t size2; + ps--; + size2 = (size + ps) & ~ps; + if (size2 >= size) + { + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (res) + return res; + } + } } #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); - if (address == 0) + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } @@ -127,10 +237,219 @@ void BigFree(void *address) #endif -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc g_Alloc = { SzAlloc, SzFree }; +static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } +static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; -static void *SzBigAlloc(void *p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } -static void SzBigFree(void *p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; + +static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } +static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} diff --git a/src/third-party/lzma/Alloc.h b/src/third-party/lzma/Alloc.h index 73b282a07..3d796e5ee 100644 --- a/src/third-party/lzma/Alloc.h +++ b/src/third-party/lzma/Alloc.h @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -29,8 +29,22 @@ void BigFree(void *address); #endif -extern ISzAlloc g_Alloc; -extern ISzAlloc g_BigAlloc; +extern const ISzAlloc g_Alloc; +extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END diff --git a/src/third-party/lzma/Bcj2.c b/src/third-party/lzma/Bcj2.c index 707362a61..da93985cf 100644 --- a/src/third-party/lzma/Bcj2.c +++ b/src/third-party/lzma/Bcj2.c @@ -1,5 +1,5 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2015-08-01 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -61,7 +61,8 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) Byte *dest = p->dest; if (dest == p->destLim) return SZ_OK; - *dest = p->temp[p->state++ - BCJ2_DEC_STATE_ORIG_0]; + *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; + p->state++; p->dest = dest + 1; } } @@ -231,10 +232,10 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) if (rem < 4) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break; diff --git a/src/third-party/lzma/Bcj2Enc.c b/src/third-party/lzma/Bcj2Enc.c index 6a210151e..7a02ecde2 100644 --- a/src/third-party/lzma/Bcj2Enc.c +++ b/src/third-party/lzma/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2014-11-10 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,7 +12,6 @@ #define PRF(x) #endif -#include #include #include "Bcj2.h" @@ -53,7 +52,7 @@ void Bcj2Enc_Init(CBcj2Enc *p) p->probs[i] = kBitModelTotal >> 1; } -static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) { if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) { @@ -166,7 +165,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) { Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - Bool needConvert; + BoolInt needConvert; p->bufs[BCJ2_STREAM_MAIN] = dest + 1; p->ip += (UInt32)num + 1; @@ -254,7 +253,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) { const Byte *src = p->src; const Byte *srcLim = p->srcLim; - unsigned finishMode = p->finishMode; + EBcj2Enc_FinishMode finishMode = p->finishMode; p->src = p->temp; p->srcLim = p->temp + p->tempPos; @@ -271,7 +270,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) unsigned i; p->tempPos = tempPos; for (i = 0; i < tempPos; i++) - p->temp[i] = p->temp[i + num]; + p->temp[i] = p->temp[(size_t)i + num]; p->src = src; p->srcLim = srcLim; diff --git a/src/third-party/lzma/Bra.c b/src/third-party/lzma/Bra.c index 976810c96..cbdcb290d 100644 --- a/src/third-party/lzma/Bra.c +++ b/src/third-party/lzma/Bra.c @@ -1,135 +1,230 @@ /* Bra.c -- Converters for RISC code -2010-04-16 : Igor Pavlov : Public domain */ +2017-04-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 8; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip += 4; + p = data; + lim = data + size; + + if (encoding) + + for (;;) { - if (data[i + 3] == 0xEB) + for (;;) { - UInt32 dest; - UInt32 src = ((UInt32)data[i + 2] << 16) | ((UInt32)data[i + 1] << 8) | (data[i + 0]); - src <<= 2; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 2; - data[i + 2] = (Byte)(dest >> 16); - data[i + 1] = (Byte)(dest >> 8); - data[i + 0] = (Byte)dest; + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v += ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v -= ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); } } - return i; } + SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 4; - for (i = 0; i <= size; i += 2) + Byte *p; + const Byte *lim; + size &= ~(size_t)1; + p = data; + lim = data + size - 4; + + if (encoding) + + for (;;) { - if ((data[i + 1] & 0xF8) == 0xF0 && - (data[i + 3] & 0xF8) == 0xF8) + UInt32 b1; + for (;;) { - UInt32 dest; - UInt32 src = - (((UInt32)data[i + 1] & 0x7) << 19) | - ((UInt32)data[i + 0] << 11) | - (((UInt32)data[i + 3] & 0x7) << 8) | - (data[i + 2]); - - src <<= 1; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 1; - - data[i + 1] = (Byte)(0xF0 | ((dest >> 19) & 0x7)); - data[i + 0] = (Byte)(dest >> 11); - data[i + 3] = (Byte)(0xF8 | ((dest >> 8) & 0x7)); - data[i + 2] = (Byte)dest; - i += 2; + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v += cur; + } + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v -= cur; + } + + /* + SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); + SetUi16(p - 2, (UInt16)(v | 0xF800)); + */ + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); } } - return i; } + SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) { - if ((data[i] >> 2) == 0x12 && (data[i + 3] & 3) == 1) + for (;;) { - UInt32 src = ((UInt32)(data[i + 0] & 3) << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3] & (~3)); - - UInt32 dest; + if (p >= lim) + return p - data; + p += 4; + /* if ((v & 0xFC000003) == 0x48000001) */ + if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + break; + } + { + UInt32 v = GetBe32(p - 4); if (encoding) - dest = ip + (UInt32)i + src; + v += ip + (UInt32)(p - data); else - dest = src - (ip + (UInt32)i); - data[i + 0] = (Byte)(0x48 | ((dest >> 24) & 0x3)); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] &= 0x3; - data[i + 3] |= dest; + v -= ip + (UInt32)(p - data); + v &= 0x03FFFFFF; + v |= 0x48000000; + SetBe32(p - 4, v); } } - return i; } + SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - UInt32 i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) - { - if ((data[i] == 0x40 && (data[i + 1] & 0xC0) == 0x00) || - (data[i] == 0x7F && (data[i + 1] & 0xC0) == 0xC0)) - { - UInt32 src = - ((UInt32)data[i + 0] << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3]); - UInt32 dest; - - src <<= 2; - if (encoding) - dest = ip + i + src; - else - dest = src - (ip + i); - dest >>= 2; - - dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) | (dest & 0x3FFFFF) | 0x40000000; + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; - data[i + 0] = (Byte)(dest >> 24); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] = (Byte)dest; + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + /* + v = GetBe32(p); + p += 4; + m = v + ((UInt32)5 << 29); + m ^= (UInt32)7 << 29; + m += (UInt32)1 << 22; + if ((m & ((UInt32)0x1FF << 23)) == 0) + break; + */ + p += 4; + if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || + (p[-4] == 0x7F && (p[-3] >= 0xC0))) + break; + } + { + UInt32 v = GetBe32(p - 4); + v <<= 2; + if (encoding) + v += ip + (UInt32)(p - data); + else + v -= ip + (UInt32)(p - data); + + v &= 0x01FFFFFF; + v -= (UInt32)1 << 24; + v ^= 0xFF000000; + v >>= 2; + v |= 0x40000000; + SetBe32(p - 4, v); } } - return i; } diff --git a/src/third-party/lzma/Bra86.c b/src/third-party/lzma/Bra86.c index 8dd3ed48d..a6463c63b 100644 --- a/src/third-party/lzma/Bra86.c +++ b/src/third-party/lzma/Bra86.c @@ -1,5 +1,5 @@ /* Bra86.c -- Converter for x86 code (BCJ) -2013-11-12 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -37,7 +37,7 @@ SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding else { mask >>= (unsigned)d; - if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(mask >> 1) + 1]))) + if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) { mask = (mask >> 1) | 4; pos++; diff --git a/src/third-party/lzma/BraIA64.c b/src/third-party/lzma/BraIA64.c index 813830c79..2656907a0 100644 --- a/src/third-party/lzma/BraIA64.c +++ b/src/third-party/lzma/BraIA64.c @@ -1,69 +1,53 @@ /* BraIA64.c -- Converter for IA-64 code -2013-11-12 : Igor Pavlov : Public domain */ +2017-01-26 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" -static const Byte kBranchTable[32] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 4, 6, 6, 0, 0, 7, 7, - 4, 4, 0, 0, 4, 4, 0, 0 -}; - SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { SizeT i; if (size < 16) return 0; size -= 16; - for (i = 0; i <= size; i += 16) + i = 0; + do { - UInt32 instrTemplate = data[i] & 0x1F; - UInt32 mask = kBranchTable[instrTemplate]; - UInt32 bitPos = 5; - int slot; - for (slot = 0; slot < 3; slot++, bitPos += 41) + unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; + if (m) { - UInt32 bytePos, bitRes; - UInt64 instruction, instNorm; - int j; - if (((mask >> slot) & 1) == 0) - continue; - bytePos = (bitPos >> 3); - bitRes = bitPos & 0x7; - instruction = 0; - for (j = 0; j < 6; j++) - instruction += (UInt64)data[i + j + bytePos] << (8 * j); - - instNorm = instruction >> bitRes; - if (((instNorm >> 37) & 0xF) == 0x5 && ((instNorm >> 9) & 0x7) == 0) + m++; + do { - UInt32 src = (UInt32)((instNorm >> 13) & 0xFFFFF); - UInt32 dest; - src |= ((UInt32)(instNorm >> 36) & 1) << 20; - - src <<= 4; - - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - - dest >>= 4; - - instNorm &= ~((UInt64)(0x8FFFFF) << 13); - instNorm |= ((UInt64)(dest & 0xFFFFF) << 13); - instNorm |= ((UInt64)(dest & 0x100000) << (36 - 20)); - - instruction &= (1 << bitRes) - 1; - instruction |= (instNorm << bitRes); - for (j = 0; j < 6; j++) - data[i + j + bytePos] = (Byte)(instruction >> (8 * j)); + Byte *p = data + (i + (size_t)m * 5 - 8); + if (((p[3] >> m) & 15) == 5 + && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) + { + unsigned raw = GetUi32(p); + unsigned v = raw >> m; + v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); + + v <<= 4; + if (encoding) + v += ip + (UInt32)i; + else + v -= ip + (UInt32)i; + v >>= 4; + + v &= 0x1FFFFF; + v += 0x700000; + v &= 0x8FFFFF; + raw &= ~((UInt32)0x8FFFFF << m); + raw |= (v << m); + SetUi32(p, raw); + } } + while (++m <= 4); } + i += 16; } + while (i <= size); return i; } diff --git a/src/third-party/lzma/Compiler.h b/src/third-party/lzma/Compiler.h index de8fab374..c788648cd 100644 --- a/src/third-party/lzma/Compiler.h +++ b/src/third-party/lzma/Compiler.h @@ -1,5 +1,5 @@ /* Compiler.h -2015-08-02 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_COMPILER_H #define __7Z_COMPILER_H @@ -21,6 +21,7 @@ #pragma warning(disable : 4514) // unreferenced inline function has been removed #pragma warning(disable : 4702) // unreachable code #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4714) // function marked as __forceinline not inlined #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information #endif diff --git a/src/third-party/lzma/CpuArch.c b/src/third-party/lzma/CpuArch.c index bcb84cb2b..ff1890e7f 100644 --- a/src/third-party/lzma/CpuArch.c +++ b/src/third-party/lzma/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2015-03-25: Igor Pavlov : Public domain */ +2018-02-18: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -45,7 +45,8 @@ static UInt32 CheckFlag(UInt32 flag) "push %%EDX\n\t" "popf\n\t" "andl %%EAX, %0\n\t": - "=c" (flag) : "c" (flag)); + "=c" (flag) : "c" (flag) : + "%eax", "%edx"); #endif return flag; } @@ -79,7 +80,13 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #else __asm__ __volatile__ ( - #if defined(MY_CPU_X86) && defined(__PIC__) + #if defined(MY_CPU_AMD64) && defined(__PIC__) + "mov %%rbx, %%rdi;" + "cpuid;" + "xchg %%rbx, %%rdi;" + : "=a" (*a) , + "=D" (*b) , + #elif defined(MY_CPU_X86) && defined(__PIC__) "mov %%ebx, %%edi;" "cpuid;" "xchgl %%ebx, %%edi;" @@ -108,7 +115,7 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #endif } -Bool x86cpuid_CheckAndRead(Cx86cpuid *p) +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) { CHECK_CPUID_IS_SUPPORTED MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); @@ -137,7 +144,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p) return -1; } -Bool CPU_Is_InOrder() +BoolInt CPU_Is_InOrder() { Cx86cpuid p; int firm; @@ -168,7 +175,7 @@ Bool CPU_Is_InOrder() #if !defined(MY_CPU_AMD64) && defined(_WIN32) #include -static Bool CPU_Sys_Is_SSE_Supported() +static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; vi.dwOSVersionInfoSize = sizeof(vi); @@ -181,7 +188,7 @@ static Bool CPU_Sys_Is_SSE_Supported() #define CHECK_SYS_SSE_SUPPORT #endif -Bool CPU_Is_Aes_Supported() +BoolInt CPU_Is_Aes_Supported() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT @@ -190,4 +197,22 @@ Bool CPU_Is_Aes_Supported() return (p.c >> 25) & 1; } +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + #endif diff --git a/src/third-party/lzma/CpuArch.h b/src/third-party/lzma/CpuArch.h index b31c2546c..5f74c1c0c 100644 --- a/src/third-party/lzma/CpuArch.h +++ b/src/third-party/lzma/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2015-12-01: Igor Pavlov : Public domain */ +2018-02-18 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -16,48 +16,122 @@ If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of pl MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. */ -#if defined(_M_X64) \ - || defined(_M_AMD64) \ - || defined(__x86_64__) \ - || defined(__AMD64__) \ - || defined(__amd64__) +#if defined(_M_X64) \ + || defined(_M_AMD64) \ + || defined(__x86_64__) \ + || defined(__AMD64__) \ + || defined(__amd64__) #define MY_CPU_AMD64 -#endif - -#if defined(MY_CPU_AMD64) \ - || defined(_M_IA64) \ - || defined(__AARCH64EL__) \ - || defined(__AARCH64EB__) + #ifdef __ILP32__ + #define MY_CPU_NAME "x32" + #else + #define MY_CPU_NAME "x64" + #endif #define MY_CPU_64BIT #endif -#if defined(_M_IX86) || defined(__i386__) -#define MY_CPU_X86 + +#if defined(_M_IX86) \ + || defined(__i386__) + #define MY_CPU_X86 + #define MY_CPU_NAME "x86" + #define MY_CPU_32BIT #endif + +#if defined(_M_ARM64) \ + || defined(__AARCH64EL__) \ + || defined(__AARCH64EB__) \ + || defined(__aarch64__) + #define MY_CPU_ARM64 + #define MY_CPU_NAME "arm64" + #define MY_CPU_64BIT +#endif + + +#if defined(_M_ARM) \ + || defined(_M_ARM_NT) \ + || defined(_M_ARMT) \ + || defined(__arm__) \ + || defined(__thumb__) \ + || defined(__ARMEL__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEL__) \ + || defined(__THUMBEB__) + #define MY_CPU_ARM + #define MY_CPU_NAME "arm" + #define MY_CPU_32BIT +#endif + + +#if defined(_M_IA64) \ + || defined(__ia64__) + #define MY_CPU_IA64 + #define MY_CPU_NAME "ia64" + #define MY_CPU_64BIT +#endif + + +#if defined(__mips64) \ + || defined(__mips64__) \ + || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) + #define MY_CPU_NAME "mips64" + #define MY_CPU_64BIT +#elif defined(__mips__) + #define MY_CPU_NAME "mips" + /* #define MY_CPU_32BIT */ +#endif + + +#if defined(__ppc64__) \ + || defined(__powerpc64__) + #ifdef __ILP32__ + #define MY_CPU_NAME "ppc64-32" + #else + #define MY_CPU_NAME "ppc64" + #endif + #define MY_CPU_64BIT +#elif defined(__ppc__) \ + || defined(__powerpc__) + #define MY_CPU_NAME "ppc" + #define MY_CPU_32BIT +#endif + + +#if defined(__sparc64__) + #define MY_CPU_NAME "sparc64" + #define MY_CPU_64BIT +#elif defined(__sparc__) + #define MY_CPU_NAME "sparc" + /* #define MY_CPU_32BIT */ +#endif + + #if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) #define MY_CPU_X86_OR_AMD64 #endif -#if defined(MY_CPU_X86) \ - || defined(_M_ARM) \ - || defined(__ARMEL__) \ - || defined(__THUMBEL__) \ - || defined(__ARMEB__) \ - || defined(__THUMBEB__) - #define MY_CPU_32BIT + +#ifdef _WIN32 + + #ifdef MY_CPU_ARM + #define MY_CPU_ARM_LE + #endif + + #ifdef MY_CPU_ARM64 + #define MY_CPU_ARM64_LE + #endif + + #ifdef _M_IA64 + #define MY_CPU_IA64_LE + #endif + #endif -#if defined(_WIN32) && defined(_M_ARM) -#define MY_CPU_ARM_LE -#endif - -#if defined(_WIN32) && defined(_M_IA64) -#define MY_CPU_IA64_LE -#endif #if defined(MY_CPU_X86_OR_AMD64) \ || defined(MY_CPU_ARM_LE) \ + || defined(MY_CPU_ARM64_LE) \ || defined(MY_CPU_IA64_LE) \ || defined(__LITTLE_ENDIAN__) \ || defined(__ARMEL__) \ @@ -66,6 +140,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem || defined(__MIPSEL__) \ || defined(__MIPSEL) \ || defined(_MIPSEL) \ + || defined(__BFIN__) \ || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) #define MY_CPU_LE #endif @@ -85,14 +160,37 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define MY_CPU_BE #endif + #if defined(MY_CPU_LE) && defined(MY_CPU_BE) -Stop_Compiling_Bad_Endian + #error Stop_Compiling_Bad_Endian #endif +#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) + #error Stop_Compiling_Bad_32_64_BIT +#endif + + +#ifndef MY_CPU_NAME + #ifdef MY_CPU_LE + #define MY_CPU_NAME "LE" + #elif defined(MY_CPU_BE) + #define MY_CPU_NAME "BE" + #else + /* + #define MY_CPU_NAME "" + */ + #endif +#endif + + + + + #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - /* || defined(__AARCH64EL__) */ + || defined(MY_CPU_ARM64) \ + || defined(__ARM_FEATURE_UNALIGNED) #define MY_CPU_LE_UNALIGN #endif #endif @@ -138,6 +236,11 @@ Stop_Compiling_Bad_Endian #endif +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif #if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) @@ -145,15 +248,21 @@ Stop_Compiling_Bad_Endian #include +#pragma intrinsic(_byteswap_ushort) #pragma intrinsic(_byteswap_ulong) #pragma intrinsic(_byteswap_uint64) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) -#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) @@ -178,10 +287,14 @@ Stop_Compiling_Bad_Endian #endif +#ifndef GetBe16 + #define GetBe16(p) ( (UInt16) ( \ ((UInt16)((const Byte *)(p))[0] << 8) | \ ((const Byte *)(p))[1] )) +#endif + #ifdef MY_CPU_X86_OR_AMD64 @@ -205,15 +318,16 @@ enum void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(); +BoolInt CPU_Is_Aes_Supported(); +BoolInt CPU_IsSupported_PageGB(); #endif diff --git a/src/third-party/lzma/DllSecur.c b/src/third-party/lzma/DllSecur.c new file mode 100644 index 000000000..19a22a9f0 --- /dev/null +++ b/src/third-party/lzma/DllSecur.c @@ -0,0 +1,108 @@ +/* DllSecur.c -- DLL loading security +2018-02-21 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#ifdef _WIN32 + +#include + +#include "DllSecur.h" + +#ifndef UNDER_CE + +typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); + +#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 +#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800 + +static const char * const g_Dlls = + #ifndef _CONSOLE + "UXTHEME\0" + #endif + "USERENV\0" + "SETUPAPI\0" + "APPHELP\0" + "PROPSYS\0" + "DWMAPI\0" + "CRYPTBASE\0" + "OLEACC\0" + "CLBCATQ\0" + "VERSION\0" + ; + +#endif + +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + +void LoadSecurityDlls() +{ + #ifndef UNDER_CE + + wchar_t buf[MAX_PATH + 100]; + + { + // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + } + + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return; + } + { + const char *dll; + unsigned pos = (unsigned)lstrlenW(buf); + + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + + for (dll = g_Dlls; dll[0] != 0;) + { + unsigned k = 0; + for (;;) + { + char c = *dll++; + buf[pos + k] = (Byte)c; + k++; + if (c == 0) + break; + } + + lstrcatW(buf, L".dll"); + LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + } + } + + #endif +} + +#endif diff --git a/src/third-party/lzma/DllSecur.h b/src/third-party/lzma/DllSecur.h new file mode 100644 index 000000000..4c113568e --- /dev/null +++ b/src/third-party/lzma/DllSecur.h @@ -0,0 +1,20 @@ +/* DllSecur.h -- DLL loading for security +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __DLL_SECUR_H +#define __DLL_SECUR_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#ifdef _WIN32 + +void My_SetDefaultDllDirectories(); +void LoadSecurityDlls(); + +#endif + +EXTERN_C_END + +#endif diff --git a/src/third-party/lzma/LzFind.c b/src/third-party/lzma/LzFind.c index c335d363c..4eefc17dd 100644 --- a/src/third-party/lzma/LzFind.c +++ b/src/third-party/lzma/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2018-07-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -16,18 +16,18 @@ #define kStartMaxLen 3 -static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc) +static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { if (!p->directInput) { - alloc->Free(alloc, p->bufferBase); + ISzAlloc_Free(alloc, p->bufferBase); p->bufferBase = NULL; } } /* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *alloc) +static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr alloc) { UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; if (p->directInput) @@ -39,7 +39,7 @@ static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *a { LzInWindow_Free(p, alloc); p->blockSize = blockSize; - p->bufferBase = (Byte *)alloc->Alloc(alloc, (size_t)blockSize); + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, (size_t)blockSize); } return (p->bufferBase != NULL); } @@ -81,7 +81,7 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) if (size == 0) return; - p->result = p->stream->Read(p->stream, dest, &size); + p->result = ISeqInStream_Read(p->stream, dest, &size); if (p->result != SZ_OK) return; if (size == 0) @@ -138,45 +138,46 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; + unsigned i; p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; MatchFinder_SetDefaultSettings(p); for (i = 0; i < 256; i++) { - UInt32 r = i; + UInt32 r = (UInt32)i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); p->crc[i] = r; } } -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc) +static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hash); + ISzAlloc_Free(alloc, p->hash); p->hash = NULL; } -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc) +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) { MatchFinder_FreeThisClassMemory(p, alloc); LzInWindow_Free(p, alloc); } -static CLzRef* AllocRefs(size_t num, ISzAlloc *alloc) +static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) { size_t sizeInBytes = (size_t)num * sizeof(CLzRef); if (sizeInBytes / sizeof(CLzRef) != num) return NULL; - return (CLzRef *)alloc->Alloc(alloc, sizeInBytes); + return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); } int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc) + ISzAllocPtr alloc) { UInt32 sizeReserv; @@ -208,7 +209,11 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, hs = (1 << 16) - 1; else { - hs = historySize - 1; + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4); @@ -292,17 +297,33 @@ static void MatchFinder_SetLimits(CMatchFinder *p) p->posLimit = p->pos + limit; } -void MatchFinder_Init_2(CMatchFinder *p, int readData) + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_3(CMatchFinder *p, int readData) { - UInt32 i; - UInt32 *hash = p->hash; - UInt32 num = p->hashSizeSum; - for (i = 0; i < num; i++) - hash[i] = kEmptyHashValue; - p->cyclicBufferPos = 0; p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; + p->pos = + p->streamPos = p->cyclicBufferSize; p->result = SZ_OK; p->streamEndWasReached = 0; @@ -312,10 +333,14 @@ void MatchFinder_Init_2(CMatchFinder *p, int readData) MatchFinder_SetLimits(p); } + void MatchFinder_Init(CMatchFinder *p) { - MatchFinder_Init_2(p, True); + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_3(p, True); } + static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) { @@ -343,6 +368,8 @@ static void MatchFinder_Normalize(CMatchFinder *p) MatchFinder_ReduceOffsets(p, subValue); } + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { if (p->pos == kMaxValForNormalize) @@ -354,10 +381,16 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + UInt32 *distances, unsigned maxLen) { + /* son[_cyclicBufferPos] = curMatch; for (;;) { @@ -375,7 +408,8 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = len; *distances++ = delta - 1; if (len == lenLimit) return distances; @@ -383,15 +417,58 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, } } } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + do + { + UInt32 delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - delta; + if (cur[maxLen] == cur[maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + distances[0] = (UInt32)(lim - cur); + distances[1] = delta - 1; + return distances + 2; + } + } + { + unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + distances[0] = (UInt32)len; + distances[1] = delta - 1; + distances += 2; + } + } + } + } + } + while (--cutValue); + + return distances; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *distances, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -401,9 +478,10 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt return distances; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -412,11 +490,12 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = (UInt32)len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; + *ptr1 = pair0; *ptr0 = pair[1]; return distances; } @@ -443,9 +522,9 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -455,9 +534,9 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const return; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); if (pb[len] == cur[len]) { while (++len != lenLimit) @@ -495,13 +574,13 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const p->buffer++; \ if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return (UInt32)offset; static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) @@ -510,22 +589,22 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue #define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; + offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; #define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ ptrdiff_t diff = (ptrdiff_t)0 - d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (UInt32)(c - cur); } + maxLen = (unsigned)(c - cur); } static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; @@ -536,7 +615,7 @@ static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; @@ -547,7 +626,8 @@ UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, d2, maxLen, offset, pos; + UInt32 h2, d2, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(3) @@ -558,10 +638,10 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) d2 = pos - hash[h2]; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = pos; - hash[kFix3HashSize + hv] = pos; + (hash + kFix3HashSize)[hv] = pos; maxLen = 2; offset = 0; @@ -569,12 +649,12 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { UPDATE_maxLen - distances[0] = maxLen; + distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; offset = 2; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -584,7 +664,8 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -593,21 +674,22 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = hash[kFix4HashSize + hv]; + curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; maxLen = 0; offset = 0; if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -615,7 +697,7 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { maxLen = 3; - distances[offset + 1] = d3 - 1; + distances[(size_t)offset + 1] = d3 - 1; offset += 2; d2 = d3; } @@ -623,10 +705,10 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -649,16 +731,16 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; - curMatch = hash[kFix5HashSize + hv]; + curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; maxLen = 0; offset = 0; @@ -691,7 +773,7 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) && *(cur - d4 + 3) == *(cur + 3)) { maxLen = 4; - distances[offset + 1] = d4 - 1; + distances[(size_t)offset + 1] = d4 - 1; offset += 2; d2 = d4; } @@ -699,7 +781,7 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) { SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); @@ -716,7 +798,8 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -725,21 +808,21 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - - curMatch = hash[kFix4HashSize + hv]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; maxLen = 0; offset = 0; if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -747,7 +830,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { maxLen = 3; - distances[offset + 1] = d3 - 1; + distances[(size_t)offset + 1] = d3 - 1; offset += 2; d2 = d3; } @@ -755,7 +838,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -766,7 +849,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen < 3) maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances + offset, maxLen) - (distances)); MOVE_POS_RET } @@ -783,16 +866,16 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; - curMatch = hash[kFix5HashSize + hv]; + curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; maxLen = 0; offset = 0; @@ -825,7 +908,7 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) && *(cur - d4 + 3) == *(cur + 3)) { maxLen = 4; - distances[offset + 1] = d4 - 1; + distances[(size_t)offset + 1] = d4 - 1; offset += 2; d2 = d4; } @@ -833,7 +916,7 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -852,12 +935,12 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances, 2) - (distances)); MOVE_POS_RET } @@ -897,9 +980,9 @@ static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_HEADER(3) HASH3_CALC; hash = p->hash; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = - hash[kFix3HashSize + hv] = p->pos; + (hash + kFix3HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0); @@ -914,10 +997,10 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_HEADER(4) HASH4_CALC; hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0); @@ -933,11 +1016,11 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0); @@ -953,10 +1036,10 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_HEADER(4) HASH4_CALC; hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; MOVE_POS } @@ -973,11 +1056,11 @@ static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = p->hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; + curMatch = hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; MOVE_POS } diff --git a/src/third-party/lzma/LzFind.h b/src/third-party/lzma/LzFind.h index 2ff667377..c77added7 100644 --- a/src/third-party/lzma/LzFind.h +++ b/src/third-party/lzma/LzFind.h @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2017-06-10 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H @@ -47,6 +47,8 @@ typedef struct _CMatchFinder SRes result; UInt32 crc[256]; size_t numRefs; + + UInt64 expectedDataSize; } CMatchFinder; #define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) @@ -71,8 +73,8 @@ void MatchFinder_Construct(CMatchFinder *p); */ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc); + ISzAllocPtr alloc); +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); @@ -103,7 +105,9 @@ typedef struct _IMatchFinder void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); -void MatchFinder_Init_2(CMatchFinder *p, int readData); +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_3(CMatchFinder *p, int readData); void MatchFinder_Init(CMatchFinder *p); UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); diff --git a/src/third-party/lzma/LzFindMt.c b/src/third-party/lzma/LzFindMt.c index cb61e0953..df32146f9 100644 --- a/src/third-party/lzma/LzFindMt.c +++ b/src/third-party/lzma/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2018-12-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -33,6 +33,8 @@ static void MtSync_GetNextBlock(CMtSync *p) Event_Set(&p->canStart); Event_Wait(&p->wasStarted); + + // if (mt) MatchFinder_Init_LowHash(mt->MatchFinder); } else { @@ -155,6 +157,9 @@ static void HashThreadFunc(CMatchFinderMt *mt) UInt32 numProcessedBlocks = 0; Event_Wait(&p->canStart); Event_Set(&p->wasStarted); + + MatchFinder_Init_HighHash(mt->MatchFinder); + for (;;) { if (p->exit) @@ -205,7 +210,7 @@ static void HashThreadFunc(CMatchFinderMt *mt) if (num > kMtHashBlockSize - 2) num = kMtHashBlockSize - 2; mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); - heads[0] += num; + heads[0] = 2 + num; } mf->pos += num; mf->buffer += num; @@ -227,38 +232,57 @@ static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) #define kEmptyHashValue 0 -/* #define MFMT_GM_INLINE */ +#define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE -#define NO_INLINE MY_FAST_CALL +/* + we use size_t for _cyclicBufferPos instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ -static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) +MY_NO_INLINE +static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) { do { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; + UInt32 *_distances = ++distances; + UInt32 delta = *hash++; - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) + unsigned maxLen = (unsigned)_maxLen; + + /* + if (size > 1) { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + UInt32 delta = *hash; + if (delta < _cyclicBufferSize) { - *ptr0 = *ptr1 = kEmptyHashValue; - break; + UInt32 cyc1 = _cyclicBufferPos + 1; + CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); + Byte b = *(cur + 1 - delta); + _distances[0] = pair[0]; + _distances[1] = b; } + } + */ + if (cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + } + else + for(;;) + { { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = *pair; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -267,54 +291,66 @@ static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte * break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; + UInt32 pair1 = pair[1]; + *ptr1 = pair0; + *ptr0 = pair1; break; } } } - if (pb[len] < cur[len]) { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + UInt32 curMatch = pos - delta; + // delta = pos - *pair; + // delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31]; + if (pb[len] < cur[len]) + { + delta = pos - pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = pos - *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } } } + if (--cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } } pos++; _cyclicBufferPos++; cur++; { UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; + _distances[-1] = num; } } - while (limit > 0 && --size != 0); + while (distances < limit && --size != 0); *posRes = pos; - return limit; + return distances; } #endif + + static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 distances[1] = p->hashNumAvail; @@ -364,8 +400,10 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) #else { UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes); + curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, + distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + distances + limit, + size, &posRes) - distances); p->hashBufPos += posRes - pos; cyclicBufferPos += posRes - pos; p->buffer += posRes - pos; @@ -443,13 +481,13 @@ void MatchFinderMt_Construct(CMatchFinderMt *p) MtSync_Construct(&p->btSync); } -static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAlloc *alloc) +static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hashBuf); + ISzAlloc_Free(alloc, p->hashBuf); p->hashBuf = NULL; } -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc) +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc) { MtSync_Destruct(&p->hashSync); MtSync_Destruct(&p->btSync); @@ -472,7 +510,7 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p) } SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc) + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc) { CMatchFinder *mf = p->MatchFinder; p->historySize = historySize; @@ -480,7 +518,7 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB return SZ_ERROR_PARAM; if (!p->hashBuf) { - p->hashBuf = (UInt32 *)alloc->Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); + p->hashBuf = (UInt32 *)ISzAlloc_Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); if (!p->hashBuf) return SZ_ERROR_MEM; p->btBuf = p->hashBuf + kHashBufferSize; @@ -496,14 +534,18 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB } /* Call it after ReleaseStream / SetStream */ -void MatchFinderMt_Init(CMatchFinderMt *p) +static void MatchFinderMt_Init(CMatchFinderMt *p) { CMatchFinder *mf = p->MatchFinder; - p->btBufPos = p->btBufPosLimit = 0; - p->hashBufPos = p->hashBufPosLimit = 0; + + p->btBufPos = + p->btBufPosLimit = 0; + p->hashBufPos = + p->hashBufPosLimit = 0; /* Init without data reading. We don't want to read data in this thread */ - MatchFinder_Init_2(mf, False); + MatchFinder_Init_3(mf, False); + MatchFinder_Init_LowHash(mf); p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf); p->btNumAvailBytes = 0; @@ -591,10 +633,10 @@ static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *dista MT_HASH3_CALC curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; + curMatch3 = (hash + kFix3HashSize)[h3]; hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; + (hash + kFix3HashSize)[h3] = lzPos; if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) { @@ -627,12 +669,12 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distan MT_HASH4_CALC curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; - curMatch4 = hash[kFix4HashSize + h4]; + curMatch3 = (hash + kFix3HashSize)[h3]; + curMatch4 = (hash + kFix4HashSize)[h4]; hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; - hash[kFix4HashSize + h4] = lzPos; + (hash + kFix3HashSize)[h3] = lzPos; + (hash + kFix4HashSize)[h4] = lzPos; if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) { @@ -684,8 +726,12 @@ static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances) UInt32 i; for (i = 0; i < len; i += 2) { - *distances++ = *btBuf++; - *distances++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances[0] = v0; + distances[1] = v1; + distances += 2; } } INCREASE_LZ_POS @@ -712,8 +758,12 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); do { - *distances2++ = *btBuf++; - *distances2++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances2[0] = v0; + distances2[1] = v1; + distances2 += 2; } while ((len -= 2) != 0); len = (UInt32)(distances2 - (distances)); @@ -746,7 +796,7 @@ static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) SKIP_HEADER_MT(3) UInt32 h2, h3; MT_HASH3_CALC - hash[kFix3HashSize + h3] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT @@ -758,8 +808,8 @@ static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) SKIP_HEADER_MT(4) UInt32 h2, h3, h4; MT_HASH4_CALC - hash[kFix4HashSize + h4] = - hash[kFix3HashSize + h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT @@ -777,7 +827,7 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) { case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)0; + p->MixMatchesFunc = (Mf_Mix_Matches)NULL; vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break; diff --git a/src/third-party/lzma/LzFindMt.h b/src/third-party/lzma/LzFindMt.h index 46b6924ad..fdd17008c 100644 --- a/src/third-party/lzma/LzFindMt.h +++ b/src/third-party/lzma/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2015-05-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -19,10 +19,10 @@ EXTERN_C_BEGIN typedef struct _CMtSync { - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; + BoolInt wasCreated; + BoolInt needStart; + BoolInt exit; + BoolInt stopWriting; CThread thread; CAutoResetEvent canStart; @@ -30,8 +30,8 @@ typedef struct _CMtSync CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; + BoolInt csWasInitialized; + BoolInt csWasEntered; CCriticalSection cs; UInt32 numProcessedBlocks; } CMtSync; @@ -90,9 +90,9 @@ typedef struct _CMatchFinderMt } CMatchFinderMt; void MatchFinderMt_Construct(CMatchFinderMt *p); -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc); +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc); SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc); + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc); void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable); void MatchFinderMt_ReleaseStream(CMatchFinderMt *p); diff --git a/src/third-party/lzma/Lzma2Dec.c b/src/third-party/lzma/Lzma2Dec.c index b6884571c..2e631051b 100644 --- a/src/third-party/lzma/Lzma2Dec.c +++ b/src/third-party/lzma/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2015-11-09 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -14,28 +14,22 @@ #include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) @@ -74,14 +68,14 @@ static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) return SZ_OK; } -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props)); return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); } -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props)); @@ -91,9 +85,11 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); } @@ -102,19 +98,26 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); - if (p->control == 0) + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); + if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if ((p->control & 0x7F) > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else - p->unpackSize = (UInt32)(p->control & 0x1F) << 16; + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; + p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0: @@ -124,8 +127,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8; @@ -134,9 +137,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: { @@ -145,13 +148,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } } @@ -167,7 +169,8 @@ static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT s p->processedPos += (UInt32)size; } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); + SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) @@ -176,12 +179,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, *srcLen = 0; *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->state != LZMA2_STATE_FINISHED) + while (p->state != LZMA2_STATE_ERROR) { - SizeT dicPos = p->decoder.dicPos; + SizeT dicPos; + + if (p->state == LZMA2_STATE_FINISHED) + { + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } - if (p->state == LZMA2_STATE_ERROR) - return SZ_ERROR_DATA; + dicPos = p->decoder.dicPos; if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) { @@ -198,29 +206,25 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, } (*srcLen)++; p->state = Lzma2Dec_UpdateState(p, *src++); - if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + break; continue; } { - SizeT destSizeCur = dicLimit - dicPos; - SizeT srcSizeCur = inSize - *srcLen; + SizeT inCur = inSize - *srcLen; + SizeT outCur = dicLimit - dicPos; ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; - if (p->unpackSize <= destSizeCur) + if (outCur >= p->unpackSize) { - destSizeCur = (SizeT)p->unpackSize; + outCur = (SizeT)p->unpackSize; curFinishMode = LZMA_FINISH_END; } if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (*srcLen == inSize) + if (inCur == 0) { *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; @@ -228,135 +232,241 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - p->needInitDic = False; + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); LzmaDec_InitDicAndState(&p->decoder, initDic, False); } - if (srcSizeCur > destSizeCur) - srcSizeCur = destSizeCur; + if (inCur > outCur) + inCur = outCur; + if (inCur == 0) + break; - if (srcSizeCur == 0) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur); - LzmaDec_UpdateWithUncompressed(&p->decoder, src, srcSizeCur); - - src += srcSizeCur; - *srcLen += srcSizeCur; - p->unpackSize -= (UInt32)srcSizeCur; + src += inCur; + *srcLen += inCur; + p->unpackSize -= (UInt32)inCur; p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; } else { - SizeT outSizeProcessed; SRes res; if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } - if (srcSizeCur > p->packSize) - srcSizeCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + destSizeCur, src, &srcSizeCur, curFinishMode, status); + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; - src += srcSizeCur; - *srcLen += srcSizeCur; - p->packSize -= (UInt32)srcSizeCur; + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); - outSizeProcessed = p->decoder.dicPos - dicPos; - p->unpackSize -= (UInt32)outSizeProcessed; + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + outCur = p->decoder.dicPos - dicPos; + p->unpackSize -= (UInt32)outCur; - RINOK(res); + if (res != 0) + break; + if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) - return res; + { + if (p->packSize == 0) + break; + return SZ_OK; + } - if (srcSizeCur == 0 && outSizeProcessed == 0) + if (inCur == 0 && outCur == 0) { if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK || p->unpackSize != 0 || p->packSize != 0) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + break; p->state = LZMA2_STATE_CONTROL; } - if (*status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) - *status = LZMA_STATUS_NOT_FINISHED; + *status = LZMA_STATUS_NOT_SPECIFIED; } } } - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + *status = LZMA_STATUS_NOT_SPECIFIED; + p->state = LZMA2_STATE_ERROR; + return SZ_ERROR_DATA; } + + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; +} + + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; *srcLen = *destLen = 0; + for (;;) { - SizeT srcSizeCur = inSize, outSizeCur, dicPos; + SizeT inCur = inSize, outCur, dicPos; ELzmaFinishMode curFinishMode; SRes res; + if (p->decoder.dicPos == p->decoder.dicBufSize) p->decoder.dicPos = 0; dicPos = p->decoder.dicPos; - if (outSize > p->decoder.dicBufSize - dicPos) + curFinishMode = LZMA_FINISH_ANY; + outCur = p->decoder.dicBufSize - dicPos; + + if (outCur >= outSize) { - outSizeCur = p->decoder.dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else - { - outSizeCur = dicPos + outSize; + outCur = outSize; curFinishMode = finishMode; } - res = Lzma2Dec_DecodeToDic(p, outSizeCur, src, &srcSizeCur, curFinishMode, status); - src += srcSizeCur; - inSize -= srcSizeCur; - *srcLen += srcSizeCur; - outSizeCur = p->decoder.dicPos - dicPos; - memcpy(dest, p->decoder.dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; + res = Lzma2Dec_DecodeToDic(p, dicPos + outCur, src, &inCur, curFinishMode, status); + + src += inCur; + inSize -= inCur; + *srcLen += inCur; + outCur = p->decoder.dicPos - dicPos; + memcpy(dest, p->decoder.dic + dicPos, outCur); + dest += outCur; + outSize -= outCur; + *destLen += outCur; if (res != 0) return res; - if (outSizeCur == 0 || outSize == 0) + if (outCur == 0 || outSize == 0) return SZ_OK; } } + SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc) + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc) { CLzma2Dec p; SRes res; diff --git a/src/third-party/lzma/Lzma2Dec.h b/src/third-party/lzma/Lzma2Dec.h index 026cdefe9..da5038725 100644 --- a/src/third-party/lzma/Lzma2Dec.h +++ b/src/third-party/lzma/Lzma2Dec.h @@ -1,5 +1,5 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2015-05-13 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H @@ -12,25 +12,24 @@ EXTERN_C_BEGIN typedef struct { - CLzmaDec decoder; - UInt32 packSize; - UInt32 unpackSize; unsigned state; Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode: It has meaning only if the decoding reaches output limit (*destLen or dicLimit). @@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /* @@ -73,7 +113,7 @@ Returns: */ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc); + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END diff --git a/src/third-party/lzma/Lzma2DecMt.c b/src/third-party/lzma/Lzma2DecMt.c new file mode 100644 index 000000000..87d5567ad --- /dev/null +++ b/src/third-party/lzma/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/src/third-party/lzma/Lzma2DecMt.h b/src/third-party/lzma/Lzma2DecMt.h new file mode 100644 index 000000000..96f89a3ca --- /dev/null +++ b/src/third-party/lzma/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/src/third-party/lzma/Lzma2Enc.c b/src/third-party/lzma/Lzma2Enc.c index cba013497..d54147752 100644 --- a/src/third-party/lzma/Lzma2Enc.c +++ b/src/third-party/lzma/Lzma2Enc.c @@ -1,9 +1,8 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2015-10-04 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" -/* #include */ #include /* #define _7ZIP_ST */ @@ -13,7 +12,7 @@ #ifndef _7ZIP_ST #include "MtCoder.h" #else -#define NUM_MT_CODER_THREADS_MAX 1 +#define MTCODER__THREADS_MAX 1 #endif #define LZMA2_CONTROL_LZMA (1 << 7) @@ -35,41 +34,97 @@ #define PRF(x) /* x */ + +/* ---------- CLimitedSeqInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + UInt64 limit; + UInt64 processed; + int finished; +} CLimitedSeqInStream; + +static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->finished = 0; +} + +static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->finished = (size2 == 0 ? 1 : 0); + p->processed += size2; + } + *size = size2; + return res; +} + + /* ---------- CLzma2EncInt ---------- */ typedef struct { CLzmaEncHandle enc; + Byte propsAreSet; + Byte propsByte; + Byte needInitState; + Byte needInitProp; UInt64 srcPos; - Byte props; - Bool needInitState; - Bool needInitProp; } CLzma2EncInt; -static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) + +static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) { - Byte propsEncoded[LZMA_PROPS_SIZE]; - SizeT propsSize = LZMA_PROPS_SIZE; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); - p->srcPos = 0; - p->props = propsEncoded[0]; - p->needInitState = True; - p->needInitProp = True; + if (!p->propsAreSet) + { + SizeT propsSize = LZMA_PROPS_SIZE; + Byte propsEncoded[LZMA_PROPS_SIZE]; + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + p->propsByte = propsEncoded[0]; + p->propsAreSet = True; + } return SZ_OK; } +static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) +{ + p->srcPos = 0; + p->needInitState = True; + p->needInitProp = True; +} + + SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig); + ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); void LzmaEnc_Finish(CLzmaEncHandle pp); void LzmaEnc_SaveState(CLzmaEncHandle pp); void LzmaEnc_RestoreState(CLzmaEncHandle pp); +/* +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +*/ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t *packSizeRes, ISeqOutStream *outStream) @@ -78,7 +133,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t packSize = packSizeLimit; UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - Bool useCopyBlock; + BoolInt useCopyBlock; SRes res; *packSizeRes = 0; @@ -126,7 +181,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, if (outStream) { *packSizeRes += destPos; - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; destPos = 0; } @@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, outBuf[destPos++] = (Byte)pm; if (p->needInitProp) - outBuf[destPos++] = p->props; + outBuf[destPos++] = p->propsByte; p->needInitProp = False; p->needInitState = False; @@ -162,7 +217,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, p->srcPos += unpackSize; if (outStream) - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; *packSizeRes = destPos; @@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, void Lzma2EncProps_Init(CLzma2EncProps *p) { LzmaEncProps_Init(&p->lzmaProps); + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; p->numTotalThreads = -1; - p->numBlockThreads = -1; - p->blockSize = 0; } void Lzma2EncProps_Normalize(CLzma2EncProps *p) { - int t1, t1n, t2, t3; + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; { CLzmaEncProps lzmaProps = p->lzmaProps; LzmaEncProps_Normalize(&lzmaProps); @@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) } t1 = p->lzmaProps.numThreads; - t2 = p->numBlockThreads; + t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; if (t3 <= 0) { @@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t1 = 1; t2 = t3; } - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; } else if (t1 <= 0) { @@ -225,46 +282,71 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) p->lzmaProps.numThreads = t1; + t2r = t2; + + fileSize = p->lzmaProps.reduceSize; + + if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzmaProps.reduceSize = p->blockSize; + LzmaEncProps_Normalize(&p->lzmaProps); + p->lzmaProps.reduceSize = fileSize; + t1 = p->lzmaProps.numThreads; - if (p->blockSize == 0) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) { - UInt32 dictSize = p->lzmaProps.dictSize; - UInt64 blockSize = (UInt64)dictSize << 2; - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dictSize) blockSize = dictSize; - p->blockSize = (size_t)blockSize; + t2r = t2 = 1; + t3 = t1; } - - if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1) + else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) { - UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; - if (temp > p->lzmaProps.reduceSize) + /* if there is no block multi-threading, we use SOLID block */ + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + } + else + { + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) { - UInt64 numBlocks = temp / p->blockSize; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt32 dictSize = p->lzmaProps.dictSize; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + p->blockSize = blockSize; + } + + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; if (numBlocks < (unsigned)t2) { - t2 = (unsigned)numBlocks; - if (t2 == 0) - t2 = 1; - t3 = t1 * t2; + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; } } } - p->numBlockThreads = t2; + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; p->numTotalThreads = t3; } static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) { - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; + return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; } @@ -274,180 +356,112 @@ typedef struct { Byte propEncoded; CLzma2EncProps props; + UInt64 expectedDataSize; - Byte *outBuf; + Byte *tempBufLzma; - ISzAlloc *alloc; - ISzAlloc *allocBig; + ISzAllocPtr alloc; + ISzAllocPtr allocBig; - CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; + CLzma2EncInt coders[MTCODER__THREADS_MAX]; #ifndef _7ZIP_ST + + ISeqOutStream *outStream; + Byte *outBuf; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ + size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; + Byte *outBufs[MTCODER__BLOCKS_MAX]; + #endif } CLzma2Enc; -/* ---------- Lzma2EncThread ---------- */ -static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt64 packTotal = 0; - SRes res = SZ_OK; - - if (!mainEncoder->outBuf) - { - mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); - if (!mainEncoder->outBuf) - return SZ_ERROR_MEM; - } - - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - for (;;) - { - size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; - res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); - if (res != SZ_OK) - break; - packTotal += packSize; - res = Progress(progress, p->srcPos, packTotal); - if (res != SZ_OK) - break; - if (packSize == 0) - break; - } - - LzmaEnc_Finish(p->enc); - - if (res == SZ_OK) - { - Byte b = 0; - if (outStream->Write(outStream, &b, 1) != 1) - return SZ_ERROR_WRITE; - } - - return res; -} - - -#ifndef _7ZIP_ST - -typedef struct -{ - IMtCoderCallback funcTable; - CLzma2Enc *lzma2Enc; -} CMtCallbackImp; - -static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, - const Byte *src, size_t srcSize, int finished) -{ - CMtCallbackImp *imp = (CMtCallbackImp *)pp; - CLzma2Enc *mainEncoder = imp->lzma2Enc; - CLzma2EncInt *p = &mainEncoder->coders[index]; - - SRes res = SZ_OK; - { - size_t destLim = *destSize; - *destSize = 0; - - if (srcSize != 0) - { - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - - RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - while (p->srcPos < srcSize) - { - size_t packSize = destLim - *destSize; - res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); - if (res != SZ_OK) - break; - *destSize += packSize; - - if (packSize == 0) - { - res = SZ_ERROR_FAIL; - break; - } - - if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - - LzmaEnc_Finish(p->enc); - if (res != SZ_OK) - return res; - } - - if (finished) - { - if (*destSize == destLim) - return SZ_ERROR_OUTPUT_EOF; - dest[(*destSize)++] = 0; - } - } - return res; -} - -#endif - - -/* ---------- Lzma2Enc ---------- */ - -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); + CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); if (!p) return NULL; Lzma2EncProps_Init(&p->props); Lzma2EncProps_Normalize(&p->props); - p->outBuf = 0; + p->expectedDataSize = (UInt64)(Int64)-1; + p->tempBufLzma = NULL; p->alloc = alloc; p->allocBig = allocBig; { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->coders[i].enc = 0; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].enc = NULL; } #ifndef _7ZIP_ST - MtCoder_Construct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } #endif return p; } + +#ifndef _7ZIP_ST + +static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + +#endif + + void Lzma2Enc_Destroy(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + for (i = 0; i < MTCODER__THREADS_MAX; i++) { CLzma2EncInt *t = &p->coders[i]; if (t->enc) { LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); - t->enc = 0; + t->enc = NULL; } } + #ifndef _7ZIP_ST - MtCoder_Destruct(&p->mtCoder); + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + Lzma2Enc_FreeOutBufs(p); #endif - IAlloc_Free(p->alloc, p->outBuf); - IAlloc_Free(p->alloc, pp); + ISzAlloc_Free(p->alloc, p->tempBufLzma); + p->tempBufLzma = NULL; + + ISzAlloc_Free(p->alloc, pp); } + SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -460,6 +474,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) return SZ_OK; } + +void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -471,50 +493,311 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) return (Byte)i; } -SRes Lzma2Enc_Encode(CLzma2EncHandle pp, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - int i; - for (i = 0; i < p->props.numBlockThreads; i++) +static SRes Lzma2Enc_EncodeMt1( + CLzma2Enc *me, + CLzma2EncInt *p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + int finished, + ICompressProgress *progress) +{ + UInt64 unpackTotal = 0; + UInt64 packTotal = 0; + size_t outLim = 0; + CLimitedSeqInStream limitedInStream; + + if (outBuf) { - CLzma2EncInt *t = &p->coders[(unsigned)i]; - if (!t->enc) + outLim = *outBufSize; + *outBufSize = 0; + } + + if (!p->enc) + { + p->propsAreSet = False; + p->enc = LzmaEnc_Create(me->alloc); + if (!p->enc) + return SZ_ERROR_MEM; + } + + limitedInStream.realStream = inStream; + if (inStream) + { + limitedInStream.vt.Read = LimitedSeqInStream_Read; + } + + if (!outBuf) + { + // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma + if (!me->tempBufLzma) { - t->enc = LzmaEnc_Create(p->alloc); - if (!t->enc) + me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); + if (!me->tempBufLzma) return SZ_ERROR_MEM; } } - #ifndef _7ZIP_ST - if (p->props.numBlockThreads > 1) - { - CMtCallbackImp mtCallback; + RINOK(Lzma2EncInt_InitStream(p, &me->props)); - mtCallback.funcTable.Code = MtCallbackImp_Code; - mtCallback.lzma2Enc = p; + for (;;) + { + SRes res = SZ_OK; + size_t inSizeCur = 0; + + Lzma2EncInt_InitBlock(p); + LimitedSeqInStream_Init(&limitedInStream); + limitedInStream.limit = me->props.blockSize; + + if (inStream) + { + UInt64 expected = (UInt64)(Int64)-1; + // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize + if (me->expectedDataSize != (UInt64)(Int64)-1 + && me->expectedDataSize >= unpackTotal) + expected = me->expectedDataSize - unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && expected > me->props.blockSize) + expected = (size_t)me->props.blockSize; + + LzmaEnc_SetDataSize(p->enc, expected); + + RINOK(LzmaEnc_PrepareForLzma2(p->enc, + &limitedInStream.vt, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + else + { + inSizeCur = inDataSize - (size_t)unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && inSizeCur > me->props.blockSize) + inSizeCur = (size_t)me->props.blockSize; + + // LzmaEnc_SetDataSize(p->enc, inSizeCur); + + RINOK(LzmaEnc_MemPrepare(p->enc, + inData + (size_t)unpackTotal, inSizeCur, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + + for (;;) + { + size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; + if (outBuf) + packSize = outLim - (size_t)packTotal; + + res = Lzma2EncInt_EncodeSubblock(p, + outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, + outBuf ? NULL : outStream); + + if (res != SZ_OK) + break; + + packTotal += packSize; + if (outBuf) + *outBufSize = (size_t)packTotal; + + res = Progress(progress, unpackTotal + p->srcPos, packTotal); + if (res != SZ_OK) + break; + + /* + if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) + break; + */ + + if (packSize == 0) + break; + } + + LzmaEnc_Finish(p->enc); + + unpackTotal += p->srcPos; + + RINOK(res); + + if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) + return SZ_ERROR_FAIL; + + if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) + { + if (finished) + { + if (outBuf) + { + size_t destPos = *outBufSize; + if (destPos >= outLim) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos] = 0; + *outBufSize = destPos + 1; + } + else + { + Byte b = 0; + if (ISeqOutStream_Write(outStream, &b, 1) != 1) + return SZ_ERROR_WRITE; + } + } + return SZ_OK; + } + } +} + + + +#ifndef _7ZIP_ST + +static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t destSize = me->outBufSize; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + me->outBufsDataSizes[outBufIndex] = 0; + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.inSize = 0; + progressThunk.outSize = 0; + + res = Lzma2Enc_EncodeMt1(me, + &me->coders[coderIndex], + NULL, dest, &destSize, + NULL, src, srcSize, + finished, + &progressThunk.vt); + + me->outBufsDataSizes[outBufIndex] = destSize; + + return res; +} + + +static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t size = me->outBufsDataSizes[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + if (me->outStream) + return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; + + if (size > me->outBuf_Rem) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBuf_Rem -= size; + me->outBuf += size; + return SZ_OK; +} + +#endif + + + +SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + + if (inStream && inData) + return SZ_ERROR_PARAM; + + if (outStream && outBuf) + return SZ_ERROR_PARAM; + + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].propsAreSet = False; + } + + #ifndef _7ZIP_ST + + if (p->props.numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = Lzma2Enc_MtCallback_Code; + vt.Write = Lzma2Enc_MtCallback_Write; + + p->outStream = outStream; + p->outBuf = NULL; + p->outBuf_Rem = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBuf_Rem = *outBufSize; + *outBufSize = 0; + } + + p->mtCoder.allocBig = p->allocBig; p->mtCoder.progress = progress; p->mtCoder.inStream = inStream; - p->mtCoder.outStream = outStream; - p->mtCoder.alloc = p->alloc; - p->mtCoder.mtCallback = &mtCallback.funcTable; + p->mtCoder.inData = inData; + p->mtCoder.inDataSize = inDataSize; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + p->mtCoder.blockSize = (size_t)p->props.blockSize; + if (p->mtCoder.blockSize != p->props.blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - p->mtCoder.blockSize = p->props.blockSize; - p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; - if (p->mtCoder.destBlockSize < p->props.blockSize) { - p->mtCoder.destBlockSize = (size_t)0 - 1; - if (p->mtCoder.destBlockSize < p->props.blockSize) - return SZ_ERROR_FAIL; + size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + Lzma2Enc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - p->mtCoder.numThreads = p->props.numBlockThreads; + + p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - return MtCoder_Code(&p->mtCoder); + { + SRes res = MtCoder_Code(&p->mtCoder); + if (!outStream) + *outBufSize = p->outBuf - outBuf; + return res; + } } + #endif - return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); + + return Lzma2Enc_EncodeMt1(p, + &p->coders[0], + outStream, outBuf, outBufSize, + inStream, inData, inDataSize, + True, /* finished */ + progress); } diff --git a/src/third-party/lzma/Lzma2Enc.h b/src/third-party/lzma/Lzma2Enc.h index 061178aaf..65f2dd145 100644 --- a/src/third-party/lzma/Lzma2Enc.h +++ b/src/third-party/lzma/Lzma2Enc.h @@ -1,5 +1,5 @@ /* Lzma2Enc.h -- LZMA2 Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA2_ENC_H #define __LZMA2_ENC_H @@ -8,11 +8,15 @@ EXTERN_C_BEGIN +#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 +#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) + typedef struct { CLzmaEncProps lzmaProps; - size_t blockSize; - int numBlockThreads; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; int numTotalThreads; } CLzma2EncProps; @@ -22,40 +26,29 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p); /* ---------- CLzmaEnc2Handle Interface ---------- */ /* Lzma2Enc_* functions can return the following exit codes: -Returns: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzma2EncHandle; -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig); +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void Lzma2Enc_Destroy(CLzma2EncHandle p); SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); -SRes Lzma2Enc_Encode(CLzma2EncHandle p, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); - -/* ---------- One Call Interface ---------- */ - -/* Lzma2Encode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -/* -SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); -*/ +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress); EXTERN_C_END diff --git a/src/third-party/lzma/Lzma86Dec.c b/src/third-party/lzma/Lzma86Dec.c index 0e49d7be9..20ac5e7a9 100644 --- a/src/third-party/lzma/Lzma86Dec.c +++ b/src/third-party/lzma/Lzma86Dec.c @@ -1,5 +1,5 @@ /* Lzma86Dec.c -- LZMA + x86 (BCJ) Filter Decoder -2015-11-08 : Igor Pavlov : Public domain */ +2016-05-16 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,9 +9,6 @@ #include "Bra.h" #include "LzmaDec.h" -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } - SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize) { unsigned i; @@ -25,7 +22,6 @@ SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize) SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen) { - ISzAlloc g_Alloc = { SzAlloc, SzFree }; SRes res; int useFilter; SizeT inSizePure; diff --git a/src/third-party/lzma/Lzma86Enc.c b/src/third-party/lzma/Lzma86Enc.c index 868c726d5..8d35e6dc5 100644 --- a/src/third-party/lzma/Lzma86Enc.c +++ b/src/third-party/lzma/Lzma86Enc.c @@ -1,5 +1,5 @@ /* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -13,16 +13,12 @@ #define SZE_OUT_OVERFLOW SZE_DATA_ERROR -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } - int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, int level, UInt32 dictSize, int filterMode) { - ISzAlloc g_Alloc = { SzAlloc, SzFree }; size_t outSize2 = *destLen; Byte *filteredStream; - Bool useFilter; + BoolInt useFilter; int mainResult = SZ_ERROR_OUTPUT_EOF; CLzmaEncProps props; LzmaEncProps_Init(&props); @@ -60,7 +56,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t minSize = 0; - Bool bestIsFiltered = False; + BoolInt bestIsFiltered = False; /* passes for SZ_FILTER_AUTO: 0 - BCJ + LZMA @@ -75,7 +71,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; size_t outPropsSize = 5; SRes curRes; - Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); if (curModeIsFiltered && !bestIsFiltered) break; if (useFilter && i == 0) diff --git a/src/third-party/lzma/LzmaDec.c b/src/third-party/lzma/LzmaDec.c index 651d1f284..4d1576419 100644 --- a/src/third-party/lzma/LzmaDec.c +++ b/src/third-party/lzma/LzmaDec.c @@ -1,12 +1,13 @@ /* LzmaDec.c -- LZMA Decoder -2015-06-23 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "LzmaDec.h" - #include +/* #include "CpuArch.h" */ +#include "LzmaDec.h" + #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -18,15 +19,22 @@ #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } @@ -46,16 +54,19 @@ i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ @@ -66,25 +77,28 @@ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -98,54 +112,117 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 0x300 - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + Out: + RangeCoder is normalized Result: SZ_OK - OK SZ_ERROR_DATA - Error p->remainLen: < kMatchSpecLenStart : normal remain = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - unsigned state = p->state; +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; @@ -164,17 +241,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates) @@ -240,13 +316,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else { UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode if (checkDicSize == 0 && processedPos == 0) return SZ_ERROR_DATA; + */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob); @@ -294,14 +373,14 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte #ifdef _LZMA_SIZE_OPT { - unsigned limit, offset; + unsigned lim, offset; CLzmaProb *probLen = prob + LenChoice; IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; - limit = (1 << kLenNumLowBits); + lim = (1 << kLenNumLowBits); } else { @@ -310,19 +389,19 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = (1 << kLenNumHighBits); + offset = kLenNumLowSymbols * 2; + lim = (1 << kLenNumHighBits); } } - TREE_DECODE(probLen, limit, len); + TREE_DECODE(probLen, lim, len); len += offset; } #else @@ -331,7 +410,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -345,7 +424,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -356,7 +435,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } } @@ -376,16 +455,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else @@ -412,19 +491,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; } @@ -435,20 +515,12 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { p->dicPos = dicPos; return SZ_ERROR_DATA; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen; @@ -467,7 +539,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte curLen = ((rem < len) ? (unsigned)rem : len); pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - processedPos += curLen; + processedPos += (UInt32)curLen; len -= curLen; if (curLen <= dicBufSize - pos) @@ -475,7 +547,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte Byte *dest = dic + dicPos; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; const Byte *lim = dest + curLen; - dicPos += curLen; + dicPos += (SizeT)curLen; do *(dest) = (Byte)*(dest + src); while (++dest != lim); @@ -500,17 +572,18 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte p->buf = buf; p->range = range; p->code = code; - p->remainLen = len; + p->remainLen = (UInt32)len; p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; p->reps[1] = rep1; p->reps[2] = rep2; p->reps[3] = rep3; - p->state = state; + p->state = (UInt32)state; return SZ_OK; } +#endif static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { @@ -519,7 +592,7 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) Byte *dic = p->dic; SizeT dicPos = p->dicPos; SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; + unsigned len = (unsigned)p->remainLen; SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ SizeT rem = limit - dicPos; if (rem < len) @@ -528,8 +601,8 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; - p->processedPos += len; - p->remainLen -= len; + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; while (len != 0) { len--; @@ -540,6 +613,14 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) } } + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { do @@ -550,9 +631,13 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte UInt32 rem = p->prop.dicSize - p->processedPos; if (limit - p->dicPos > rem) limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; @@ -561,9 +646,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; - return 0; } @@ -580,17 +662,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS UInt32 range = p->range; UInt32 code = p->code; const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK @@ -618,10 +700,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); } @@ -648,7 +731,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; @@ -691,7 +774,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } @@ -702,15 +785,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } @@ -722,7 +805,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex) @@ -733,7 +816,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else { @@ -745,17 +828,18 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } } @@ -766,20 +850,19 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) @@ -788,53 +871,54 @@ void LzmaDec_Init(CLzmaDec *p) LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); *status = LZMA_STATUS_NOT_SPECIFIED; + if (p->remainLen > kMatchSpecLenStart) + { + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } + + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + while (p->remainLen != kMatchSpecLenStart) { - int checkEndMarkNow; + int checkEndMarkNow = 0; - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } - - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0) @@ -855,9 +939,6 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - if (p->tempBufSize == 0) { SizeT processed; @@ -898,10 +979,10 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->tempBufSize = rem; if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); if (dummyRes == DUMMY_ERROR) { - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } @@ -924,17 +1005,20 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr return SZ_ERROR_FAIL; /* some internal error */ lookAhead -= rem; } - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; src += lookAhead; - inSize -= lookAhead; + inSize -= (SizeT)lookAhead; p->tempBufSize = 0; } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; @@ -975,19 +1059,19 @@ SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *sr } } -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->probs); + ISzAlloc_Free(alloc, p->probs); p->probs = NULL; } -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->dic); + ISzAlloc_Free(alloc, p->dic); p->dic = NULL; } -void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) { LzmaDec_FreeProbs(p, alloc); LzmaDec_FreeDict(p, alloc); @@ -1011,29 +1095,30 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) { UInt32 numProbs = LzmaProps_GetNumProbs(propNew); if (!p->probs || numProbs != p->numProbs) { LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; + p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; RINOK(LzmaProps_Decode(&propNew, props, propsSize)); @@ -1042,7 +1127,7 @@ SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, I return SZ_OK; } -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; SizeT dicBufSize; @@ -1062,7 +1147,7 @@ SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAll if (!p->dic || dicBufSize != p->dicBufSize) { LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); + p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); if (!p->dic) { LzmaDec_FreeProbs(p, alloc); @@ -1076,7 +1161,7 @@ SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAll SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc) + ELzmaStatus *status, ISzAllocPtr alloc) { CLzmaDec p; SRes res; diff --git a/src/third-party/lzma/LzmaDec.h b/src/third-party/lzma/LzmaDec.h index 2633abeac..28ce60c3e 100644 --- a/src/third-party/lzma/LzmaDec.h +++ b/src/third-party/lzma/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2013-01-18 : Igor Pavlov : Public domain */ +2018-04-21 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -12,11 +12,13 @@ EXTERN_C_BEGIN /* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */ @@ -25,7 +27,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -47,32 +52,34 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum { @@ -129,11 +136,11 @@ LzmaDec_Allocate* can return: SZ_ERROR_UNSUPPORTED - Unsupported properties */ -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ @@ -142,7 +149,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) { @@ -220,7 +227,7 @@ Returns: SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc); + ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END diff --git a/src/third-party/lzma/LzmaEnc.c b/src/third-party/lzma/LzmaEnc.c index 11de42c09..14086fc4f 100644 --- a/src/third-party/lzma/LzmaEnc.c +++ b/src/third-party/lzma/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2015-11-08 : Igor Pavlov : Public domain */ +2019-01-10: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -23,17 +23,8 @@ static unsigned g_STAT_OFFSET = 0; #endif -#define kMaxHistorySize ((UInt32)3 << 29) -/* #define kMaxHistorySize ((UInt32)7 << 29) */ - -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) +#define kLzmaMaxHistorySize ((UInt32)3 << 29) +/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -47,6 +38,8 @@ static unsigned g_STAT_OFFSET = 0; #define kNumBitPriceShiftBits 4 #define kBitPrice (1 << kNumBitPriceShiftBits) +#define REP_LEN_COUNT 64 + void LzmaEncProps_Init(CLzmaEncProps *p) { p->level = 5; @@ -62,14 +55,15 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (level < 0) level = 5; p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); + if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); if (p->dictSize > p->reduceSize) { unsigned i; + UInt32 reduceSize = (UInt32)p->reduceSize; for (i = 11; i <= 30; i++) { - if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } + if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } + if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } } } @@ -108,11 +102,11 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) #define kDicLogSizeMaxCompress 32 -#define BSR2_RET(pos, res) { unsigned long i; _BitScanReverse(&i, (pos)); res = (i + i) + ((pos >> (i - 1)) & 1); } +#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } -static UInt32 GetPosSlot1(UInt32 pos) +static unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; + unsigned res; BSR2_RET(pos, res); return res; } @@ -145,19 +139,19 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) /* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ /* -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } + res = p->g_FastPos[pos >> zz] + (zz * 2); } */ /* -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } + res = p->g_FastPos[pos >> zz] + (zz * 2); } */ -#define BSR2_RET(pos, res) { UInt32 i = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ - res = p->g_FastPos[pos >> i] + (i * 2); } +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } /* #define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ @@ -167,32 +161,35 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) #define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } #endif #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { UInt32 price; - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; } COptimal; -#define kNumOpts (1 << 12) + +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 @@ -200,22 +197,21 @@ typedef struct #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) - #define kNumAlignBits 4 #define kAlignTableSize (1 << kNumAlignBits) #define kAlignMask (kAlignTableSize - 1) #define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8 @@ -223,15 +219,11 @@ typedef struct #define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) @@ -241,27 +233,31 @@ typedef struct typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; typedef struct { - CLenEnc p; - UInt32 tableSize; + unsigned tableSize; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ typedef struct { UInt32 range; - Byte cache; + unsigned cache; UInt64 low; UInt64 cacheSize; Byte *buf; @@ -277,69 +273,77 @@ typedef struct { CLzmaProb *litProbs; - UInt32 state; + unsigned state; UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLenEnc lenProbs; + CLenEnc repLenProbs; + } CSaveState; +typedef UInt32 CProbPrice; + + typedef struct { void *matchFinderObj; IMatchFinder matchFinder; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + unsigned optCur; + unsigned optEnd; - UInt32 longestMatchLength; - UInt32 numPairs; + unsigned longestMatchLen; + unsigned numPairs; UInt32 numAvail; - UInt32 numFastBytes; - UInt32 additionalOffset; + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; unsigned lc, lp, pb; - unsigned lpMask, pbMask; unsigned lclp; - CLzmaProb *litProbs; - - Bool fastMode; - Bool writeEndMark; - Bool finished; - Bool multiThread; - Bool needInit; + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; UInt64 nowPos64; - UInt32 matchPriceCount; - UInt32 alignPriceCount; + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; - UInt32 distTableSize; + unsigned distTableSize; UInt32 dictSize; SRes result; - CRangeEnc rc; - #ifndef _7ZIP_ST - Bool mtMode; + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads #endif CMatchFinder matchFinderBase; @@ -348,33 +352,37 @@ typedef struct Byte pad[128]; #endif - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; + COptimal opt[kNumOpts]; + CSaveState saveState; #ifndef _7ZIP_ST @@ -383,58 +391,62 @@ typedef struct } CLzmaEnc; + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { CLzmaEnc *p = (CLzmaEnc *)pp; CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { CLzmaEnc *dest = (CLzmaEnc *)pp; const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) { CLzmaEnc *p = (CLzmaEnc *)pp; @@ -445,7 +457,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kMaxHistorySize) + || props.dictSize > kLzmaMaxHistorySize) return SZ_ERROR_PARAM; p->dictSize = props.dictSize; @@ -461,9 +473,10 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->lp = props.lp; p->pb = props.pb; p->fastMode = (props.algo == 0); + // p->_maxMode = True; p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); { - UInt32 numHashBytes = 4; + unsigned numHashBytes = 4; if (props.btMode) { if (props.numHashBytes < 2) @@ -492,13 +505,27 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) return SZ_OK; } -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; -#define IsCharState(s) ((s) < 7) +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.expectedDataSize = expectedDataSiize; +} + +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 + +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30) @@ -509,14 +536,16 @@ static void RangeEnc_Construct(CRangeEnc *p) p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) -static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) + +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { if (!p->bufBase) { - p->bufBase = (Byte *)alloc->Alloc(alloc, RC_BUF_SIZE); + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); if (!p->bufBase) return 0; p->bufLim = p->bufBase + RC_BUF_SIZE; @@ -524,19 +553,19 @@ static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) return 1; } -static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc) +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->bufBase); + ISzAlloc_Free(alloc, p->bufBase); p->bufBase = 0; } static void RangeEnc_Init(CRangeEnc *p) { /* Stream.Init(); */ - p->low = 0; p->range = 0xFFFFFFFF; - p->cacheSize = 1; p->cache = 0; + p->low = 0; + p->cacheSize = 0; p->buf = p->bufBase; @@ -544,37 +573,48 @@ static void RangeEnc_Init(CRangeEnc *p) p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { size_t num; if (p->res != SZ_OK) return; num = p->buf - p->bufBase; - if (num != p->outStream->Write(p->outStream, p->bufBase, num)) + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) p->res = SZ_ERROR_WRITE; p->processed += num; p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) { - Byte temp = p->cache; - do { Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); p->buf = buf; if (buf == p->bufLim) RangeEnc_FlushStream(p); - temp = 0xFF; + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (--p->cacheSize == 0) + return; } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; } static void RangeEnc_FlushData(CRangeEnc *p) @@ -584,78 +624,121 @@ static void RangeEnc_FlushData(CRangeEnc *p) RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } + +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) { + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; +} + +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) +{ + UInt32 range = p->range; + sym |= 0x100; do { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); } - while (numBits != 0); + while (sym < 0x10000); + p->range = range; } -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) -{ - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; - } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } -} - -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) -{ - symbol |= 0x100; - do - { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; - } - while (symbol < 0x10000); -} - -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) { + UInt32 range = p->range; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) { UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; for (j = 0; j < kCyclesBits; j++) { w = w * w; @@ -666,559 +749,655 @@ static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) bitCount++; } } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); } } -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { UInt32 price = 0; - symbol |= 0x100; + sym |= 0x100; do { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } - while (symbol < 0x10000); + while (sym >= 2); return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); } - while (symbol < 0x10000); + while (sym < 0x10000); return price; } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) + UInt32 range = rc->range; + unsigned m = 1; + do { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; } + while (--numBits); + rc->range = range; } -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) - { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } - return price; -} - -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; for (i = 0; i < kLenNumHighSymbols; i++) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { - if (symbol < kLenNumLowSymbols) + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; } - else + sym -= kLenNumLowSymbols; + } + + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); + { + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; + } +} + +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) +{ + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } +} + + +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) +{ + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } + + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } } } } -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) -{ - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); - } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); - } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); -} - -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) -{ - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; -} - -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) -{ - UInt32 posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) -{ - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - - - - -static void MovePos(CLzmaEnc *p, UInt32 num) -{ +/* #ifdef SHOW_STAT g_STAT_OFFSET += num; printf("\n MovePos %u", num); #endif +*/ - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) + +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 lenRes = 0, numPairs; + unsigned numPairs; + + p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + *numPairsRes = numPairs; #ifdef SHOW_STAT printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); g_STAT_OFFSET++; { - UInt32 i; + unsigned i; for (i = 0; i < numPairs; i += 2) printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); } #endif - if (numPairs > 0) + if (numPairs == 0) + return 0; { - lenRes = p->matches[numPairs - 2]; - if (lenRes == p->numFastBytes) + unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { UInt32 numAvail = p->numAvail; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; { - const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *pby = pbyCur + lenRes; - ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[numPairs - 1]; - const Byte *pbyLim = pbyCur + numAvail; - for (; pby != pbyLim && *pby == pby[dif]; pby++); - lenRes = (UInt32)(pby - pbyCur); + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; + ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); } } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; } +#define MARK_LIT ((UInt32)(Int32)-1) -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; + UInt32 prob = p->isRepG0[state]; if (repIndex == 0) { - price = GET_PRICE_0(p->isRepG0[state]); + price = GET_PRICE_0(prob); price += GET_PRICE_1(p->isRep0Long[state][posState]); } else { - price = GET_PRICE_1(p->isRepG0[state]); + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); + price += GET_PRICE_0(prob); else { - price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE_1(prob); price += GET_PRICE(p->isRepG2[state], repIndex - 2); } } return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) { - if (p->opt[cur].prev1IsChar) + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) { - p->opt[posMem - 1].prev1IsChar = False; - p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2; + p->opt[wr].dist = dist; + dist = MARK_LIT; } - } - { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; - } - } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; -} - -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) - -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) -{ - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, lenEnd, len, cur; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; - UInt32 *matches; - const Byte *data; - Byte curByte, matchByte; - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - if (numAvail < 2) - { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - { - repLens[i] = 0; - continue; - } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) - { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); - } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do - { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; - } - } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) - { - COptimal *opt; - UInt32 distance = matches[offs + 1]; - - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; else { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; - } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - if (len == matches[offs]) - { - offs += 2; - if (offs == numPairs) - break; + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; } } - } - cur = 0; + if (cur == 0) + { + p->backRes = dist; + p->optCur = wr; + return len; + } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; + } +} + + + +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) + + +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) +{ + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; + UInt32 *matches; + + { + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLen; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + if (numAvail < 2) + { + p->backRes = MARK_LIT; + return 1; + } + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + { + repLens[i] = 0; + continue; + } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + } + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + + if (mainLen >= p->numFastBytes) + { + p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); + + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) + { + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } + + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) + { + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); + } + if (last < 2) + { + p->backRes = p->opt[1].dist; + return 1; + } + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; + } + } + while (--repLen >= 2); + } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (len < 2) + len = 2; + else + while (len > matches[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = matches[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + } + + + cur = 0; #ifdef SHOW_STAT2 /* if (position >= 0) */ { unsigned i; printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) + for (i = cur; i <= last; i++) printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } #endif + } + + + + // ---------- Optimal Parsing ---------- for (;;) { - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; Byte curByte, matchByte; const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; + COptimal *curOpt, *nextOpt; - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) { p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + p->longestMatchLen = newLen; + break; } - position++; + curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) - { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) { + state = (unsigned)p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else @@ -1226,355 +1405,499 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } else { - UInt32 pos; const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); } else { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else state = kMatchNextStates[state]; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } } else { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i - 1]; + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; } } + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); posState = (position & p->pbMask); - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + + { + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); + } + + nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; + + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else { const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - nextOpt = &p->opt[cur + 1]; - - if (curAnd1Price < nextOpt->price) - { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; - } - - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) + + if (litPrice < nextOpt->price) { - nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; } } + + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); + numAvailFull = p->numAvail; { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; } + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new + { + nextOpt->price = shortRepPrice; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; + } + } + if (numAvailFull < 2) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - if (!nextIsChar && matchByte != curByte) /* speed optimization */ - { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - reps[0] - 1; - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; + // numAvail <= p->numFastBytes - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) + // ---------- LIT : REP_0 ---------- + + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) + { + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); } } } } startLen = 2; /* speed optimization */ + { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - reps[repIndex] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; - } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; - /* if (_maxMode) */ + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; - if (limit > numAvailFull) - limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do { - UInt32 nextRepMatchPrice; - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; - } + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; } } + while (--len2 >= 2); } + + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ + { + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { + { + unsigned offset = cur + len + len2; + + if (last < offset) + last = offset; + // do + { + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; + } + } + // while (len2 >= 3); + } + } + } + } + } } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ if (newLen > numAvail) { newLen = numAvail; for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; + matches[numPairs] = (UInt32)newLen; numPairs += 2; } + + // startLen = 2; /* speed optimization */ + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } offs = 0; while (startLen > matches[offs]) offs += 2; - curBack = matches[offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) + dist = matches[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } } - if (/*_maxMode && */lenTest == matches[offs]) + if (len == matches[offs]) { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - curBack - 1; - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice; + UInt32 price2; COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; } } + // while (len2 >= 3); } + + } + offs += 2; if (offs == numPairs) break; - curBack = matches[offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + dist = matches[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); } } } } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; const Byte *data; - const UInt32 *matches; if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; + mainLen = p->longestMatchLen; numPairs = p->numPairs; } numAvail = p->numAvail; - *backRes = (UInt32)-1; + p->backRes = MARK_LIT; if (numAvail < 2) return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} if (len >= p->numFastBytes) { - *backRes = i; - MovePos(p, len - 1); + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) return len; } if (len > repLen) @@ -1584,84 +1907,152 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) } } - matches = p->matches; if (mainLen >= p->numFastBytes) { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; } mainDist = 0; /* for GCC */ + if (mainLen >= 2) { - mainDist = matches[numPairs - 1]; - while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1) + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) { - if (!ChangePair(matches[numPairs - 3], mainDist)) + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) break; numPairs -= 2; - mainLen = matches[numPairs - 2]; - mainDist = matches[numPairs - 1]; + mainLen--; + mainDist = dist2; } if (mainLen == 2 && mainDist >= 0x80) mainLen = 1; } - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - *backRes = repIndex; - MovePos(p, repLen - 1); + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) return repLen; } if (mainLen < 2 || numAvail <= 2) return 1; - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) { - UInt32 newDistance = matches[p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } } data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len, limit; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) { if (p->result != SZ_OK) @@ -1675,7 +2066,8 @@ static SRes CheckErrors(CLzmaEnc *p) return p->result; } -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True; @@ -1686,48 +2078,126 @@ static SRes Flush(CLzmaEnc *p, UInt32 nowPos) return CheckErrors(p); } -static void FillAlignPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } } -static void FillDistancesPrices(CLzmaEnc *p) -{ - UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) - { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); - } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) +{ + // int y; for (y = 0; y < 100; y++) { + + UInt32 tempPrices[kNumFullDistances]; + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - UInt32 i; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); } } - p->matchPriceCount = 0; + + for (lps = 0; lps < kNumLenToPosStates; lps++) + { + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } + } + + { + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } + } + } + // } } + + void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); @@ -1751,26 +2221,27 @@ void LzmaEnc_Construct(CLzmaEnc *p) LzmaEnc_InitPriceTables(p->ProbPrices); p->litProbs = NULL; p->saveState.litProbs = NULL; + } -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc) +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) { void *p; - p = alloc->Alloc(alloc, sizeof(CLzmaEnc)); + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); if (p) LzmaEnc_Construct((CLzmaEnc *)p); return p; } -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) +void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->litProbs); - alloc->Free(alloc, p->saveState.litProbs); + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); p->litProbs = NULL; p->saveState.litProbs = NULL; } -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) +void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { #ifndef _7ZIP_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); @@ -1781,13 +2252,14 @@ void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) RangeEnc_Free(&p->rc, alloc); } -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig) +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - alloc->Free(alloc, p); + ISzAlloc_Free(alloc, p); } -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit) @@ -1805,13 +2277,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize if (p->nowPos64 == 0) { - UInt32 numPairs; + unsigned numPairs; Byte curByte; if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) return Flush(p, nowPos32); ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); LitEnc_Encode(&p->rc, p->litProbs, curByte); p->additionalOffset--; @@ -1819,123 +2291,253 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) { - UInt32 pos, len, posState; - + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + if (p->fastMode) - len = GetOptimumFast(p, &pos); + len = GetOptimumFast(p); else - len = GetOptimum(p, nowPos32, &pos); + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); #endif - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) + if (dist == MARK_LIT) { Byte curByte; - CLzmaProb *probs; const Byte *data; + unsigned state; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + RC_BIT_0(&p->rc, probs); + p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) LitEnc_Encode(&p->rc, probs, curByte); else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } else { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; p->reps[3] = p->reps[2]; + } p->reps[2] = p->reps[1]; } p->reps[1] = p->reps[0]; - p->reps[0] = distance; + p->reps[0] = dist; } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; p->state = kRepNextStates[p->state]; } } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); - - if (posSlot >= kStartPosModelIndex) - { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); - else - { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; - } - } + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; p->reps[3] = p->reps[2]; p->reps[2] = p->reps[1]; p->reps[1] = p->reps[0]; - p->reps[0] = pos; + p->reps[0] = dist + 1; + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } + + if (dist >= kStartPosModelIndex) + { + unsigned footerBits = ((posSlot >> 1) - 1); + + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } + else + { + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } + } + } } } + + nowPos32 += (UInt32)len; p->additionalOffset -= len; - nowPos32 += len; + if (p->additionalOffset == 0) { UInt32 processed; + if (!p->fastMode) { - if (p->matchPriceCount >= (1 << 7)) - FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) + /* + if (p->alignPriceCount >= 16) // kAlignTableSize FillAlignPrices(p); + if (p->matchPriceCount >= 128) + FillDistancesPrices(p); + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { + FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + } } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) break; processed = nowPos32 - startPos32; - if (useLimits) + + if (maxPackSize) { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) break; } else if (processed >= (1 << 17)) @@ -1945,13 +2547,16 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } } } + p->nowPos64 += nowPos32 - startPos32; return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { UInt32 beforeSize = kNumOpts; if (!RangeEnc_Alloc(&p->rc, alloc)) @@ -1966,8 +2571,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) { LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); if (!p->litProbs || !p->saveState.litProbs) { LzmaEnc_FreeLits(p, alloc); @@ -1985,8 +2590,13 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I #ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, + LZMA_MATCH_LEN_MAX + + 1 /* 18.04 */ + , allocBig)); p->matchFinderObj = &p->matchFinderMt; + p->matchFinderBase.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else @@ -2003,17 +2613,21 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I void LzmaEnc_Init(CLzmaEnc *p) { - UInt32 i; + unsigned i; p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; RangeEnc_Init(&p->rc); + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; for (i = 0; i < kNumStates; i++) { - UInt32 j; + unsigned j; for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) { p->isMatch[i][j] = kProbInitValue; @@ -2025,41 +2639,47 @@ void LzmaEnc_Init(CLzmaEnc *p) p->isRepG2[i] = kProbInitValue; } - { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - CLzmaProb *probs = p->litProbs; - for (i = 0; i < num; i++) - probs[i] = kProbInitValue; - } - { for (i = 0; i < kNumLenToPosStates; i++) { CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; + unsigned j; for (j = 0; j < (1 << kNumPosSlotBits); j++) probs[j] = kProbInitValue; } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + for (i = 0; i < kNumFullDistances; i++) p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + p->additionalOffset = 0; p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } + void LzmaEnc_InitPrices(CLzmaEnc *p) { if (!p->fastMode) @@ -2071,14 +2691,17 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); } -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) if (p->dictSize <= ((UInt32)1 << i)) break; p->distTableSize = i * 2; @@ -2093,7 +2716,7 @@ static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *a } static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; p->matchFinderBase.stream = inStream; @@ -2104,7 +2727,7 @@ static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInS SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; p->matchFinderBase.stream = inStream; @@ -2120,12 +2743,13 @@ static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) } SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; LzmaEnc_SetInputBuf(p, src, srcLen); p->needInit = 1; + LzmaEnc_SetDataSize(pp, srcLen); return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } @@ -2143,15 +2767,15 @@ void LzmaEnc_Finish(CLzmaEncHandle pp) typedef struct { - ISeqOutStream funcTable; + ISeqOutStream vt; Byte *data; SizeT rem; - Bool overflow; -} CSeqOutStreamBuf; + BoolInt overflow; +} CLzmaEnc_SeqOutStreamBuf; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp; + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); if (p->rem < size) { size = p->rem; @@ -2178,15 +2802,15 @@ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) } -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { CLzmaEnc *p = (CLzmaEnc *)pp; UInt64 nowPos64; SRes res; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; @@ -2198,11 +2822,15 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, if (reInit) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); + nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + if (desiredPackSize == 0) + return SZ_ERROR_OUTPUT_EOF; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem; @@ -2225,12 +2853,12 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) for (;;) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + res = LzmaEnc_CodeOneBlock(p, 0, 0); if (res != SZ_OK || p->finished) break; if (progress) { - res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); if (res != SZ_OK) { res = SZ_ERROR_PROGRESS; @@ -2242,7 +2870,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) LzmaEnc_Finish(p); /* - if (res == S_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) res = SZ_ERROR_FAIL; } */ @@ -2252,7 +2880,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); @@ -2287,21 +2915,27 @@ SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) } +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return ((CLzmaEnc *)pp)->writeEndMark; +} + + SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { SRes res; CLzmaEnc *p = (CLzmaEnc *)pp; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; p->writeEndMark = writeEndMark; - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); @@ -2321,7 +2955,7 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); SRes res; diff --git a/src/third-party/lzma/LzmaEnc.h b/src/third-party/lzma/LzmaEnc.h index c2806b45f..c9938f04b 100644 --- a/src/third-party/lzma/LzmaEnc.h +++ b/src/third-party/lzma/LzmaEnc.h @@ -1,5 +1,5 @@ /* LzmaEnc.h -- LZMA Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H @@ -12,12 +12,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaEncProps { - int level; /* 0 <= level <= 9 */ + int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ - UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF. - Encoder uses this value to reduce dictionary size */ + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ int lc; /* 0 <= lc <= 8, default = 3 */ int lp; /* 0 <= lp <= 4, default = 0 */ int pb; /* 0 <= pb <= 4, default = 2 */ @@ -25,9 +23,12 @@ typedef struct _CLzmaEncProps int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p); @@ -37,41 +38,38 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); /* ---------- CLzmaEncHandle Interface ---------- */ -/* LzmaEnc_* functions can return the following exit codes: -Returns: +/* LzmaEnc* functions can return the following exit codes: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzmaEncHandle; -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig); +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + /* ---------- One Call Interface ---------- */ -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); EXTERN_C_END diff --git a/src/third-party/lzma/MtCoder.c b/src/third-party/lzma/MtCoder.c index 8c0d9b339..5667f2d5b 100644 --- a/src/third-party/lzma/MtCoder.c +++ b/src/third-party/lzma/MtCoder.c @@ -1,168 +1,89 @@ /* MtCoder.c -- Multi-thread Coder -2015-10-13 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -void LoopThread_Construct(CLoopThread *p) -{ - Thread_Construct(&p->thread); - Event_Construct(&p->startEvent); - Event_Construct(&p->finishedEvent); -} +#ifndef _7ZIP_ST -void LoopThread_Close(CLoopThread *p) +SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) { - Thread_Close(&p->thread); - Event_Close(&p->startEvent); - Event_Close(&p->finishedEvent); -} - -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp) -{ - CLoopThread *p = (CLoopThread *)pp; - for (;;) + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) { - if (Event_Wait(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - if (p->stop) - return 0; - p->res = p->func(p->param); - if (Event_Set(&p->finishedEvent) != 0) - return SZ_ERROR_THREAD; + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; } + if (outSize != (UInt64)(Int64)-1) + { + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; + } + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } -WRes LoopThread_Create(CLoopThread *p) + +void MtProgressThunk_CreateVTable(CMtProgressThunk *p) { - p->stop = 0; - RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent)); - RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent)); - return Thread_Create(&p->thread, LoopThreadFunc, p); + p->vt.Progress = MtProgressThunk_Progress; } -WRes LoopThread_StopAndWait(CLoopThread *p) -{ - p->stop = 1; - if (Event_Set(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - return Thread_Wait(&p->thread); -} -WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); } -WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); } - -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) -{ - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; -} - -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->inSizes[i] = p->outSizes[i] = 0; - p->totalInSize = p->totalOutSize = 0; - p->progress = progress; - p->res = SZ_OK; -} - -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - p->inSizes[index] = 0; - p->outSizes[index] = 0; -} - -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } - -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) -{ - SRes res; - CriticalSection_Enter(&p->cs); - UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize) - UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize) - if (p->res == SZ_OK) - p->res = Progress(p->progress, p->totalInSize, p->totalOutSize); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; -} - -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} - -static void MtCoder_SetError(CMtCoder* p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} - -/* ---------- MtThread ---------- */ - -void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder) -{ - p->mtCoder = mtCoder; - p->outBuf = 0; - p->inBuf = 0; - Event_Construct(&p->canRead); - Event_Construct(&p->canWrite); - LoopThread_Construct(&p->thread); -} #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static void CMtThread_CloseEvents(CMtThread *p) + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) { - Event_Close(&p->canRead); - Event_Close(&p->canWrite); + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); } -static void CMtThread_Destruct(CMtThread *p) -{ - CMtThread_CloseEvents(p); - if (Thread_WasCreated(&p->thread.thread)) +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent); + if (wres == 0) { - LoopThread_StopAndWait(&p->thread); - LoopThread_Close(&p->thread); + t->stop = False; + if (!Thread_WasCreated(&t->thread)) + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + wres = Event_Set(&t->startEvent); + } + if (wres == 0) + return SZ_OK; + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +static void MtCoderThread_Destruct(CMtCoderThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + t->stop = 1; + Event_Set(&t->startEvent); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); } - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->outBuf); - p->outBuf = 0; + Event_Close(&t->startEvent); - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->inBuf); - p->inBuf = 0; + if (t->inBuf) + { + ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf); + t->inBuf = NULL; + } } -#define MY_BUF_ALLOC(buf, size, newSize) \ - if (buf == 0 || size != newSize) \ - { IAlloc_Free(p->mtCoder->alloc, buf); \ - size = newSize; buf = (Byte *)IAlloc_Alloc(p->mtCoder->alloc, size); \ - if (buf == 0) return SZ_ERROR_MEM; } -static SRes CMtThread_Prepare(CMtThread *p) -{ - MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize) - MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize) - - p->stopReading = False; - p->stopWriting = False; - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite)); - - return SZ_OK; -} static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) { @@ -170,158 +91,511 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) *processedSize = 0; while (size != 0) { - size_t curSize = size; - SRes res = stream->Read(stream, data, &curSize); - *processedSize += curSize; - data += curSize; - size -= curSize; + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; RINOK(res); - if (curSize == 0) + if (cur == 0) return SZ_OK; } return SZ_OK; } -#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1] -static SRes MtThread_Process(CMtThread *p, Bool *stop) +/* + ThreadFunc2() returns: + SZ_OK - in all normal cases (even for stream error or memory allocation error) + SZ_ERROR_THREAD - in case of failure in system synch function +*/ + +static SRes ThreadFunc2(CMtCoderThread *t) { - CMtThread *next; - *stop = True; - if (Event_Wait(&p->canRead) != 0) - return SZ_ERROR_THREAD; - - next = GET_NEXT_THREAD(p); - - if (p->stopReading) - { - next->stopReading = True; - return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD; - } + CMtCoder *mtc = t->mtCoder; - { - size_t size = p->mtCoder->blockSize; - size_t destSize = p->outBufSize; - - RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size)); - next->stopReading = *stop = (size != p->mtCoder->blockSize); - if (Event_Set(&next->canRead) != 0) - return SZ_ERROR_THREAD; - - RINOK(p->mtCoder->mtCallback->Code(p->mtCoder->mtCallback, p->index, - p->outBuf, &destSize, p->inBuf, size, *stop)); - - MtProgress_Reinit(&p->mtCoder->mtProgress, p->index); - - if (Event_Wait(&p->canWrite) != 0) - return SZ_ERROR_THREAD; - if (p->stopWriting) - return SZ_ERROR_FAIL; - if (p->mtCoder->outStream->Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize) - return SZ_ERROR_WRITE; - return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD; - } -} - -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) -{ - CMtThread *p = (CMtThread *)pp; for (;;) { - Bool stop; - CMtThread *next = GET_NEXT_THREAD(p); - SRes res = MtThread_Process(p, &stop); - if (res != SZ_OK) + unsigned bi; + SRes res; + SRes res2; + BoolInt finished; + unsigned bufIndex; + size_t size; + const Byte *inData; + UInt64 readProcessed = 0; + + RINOK_THREAD(Event_Wait(&mtc->readEvent)) + + /* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */ + + if (mtc->stopReading) { - MtCoder_SetError(p->mtCoder, res); - MtProgress_SetError(&p->mtCoder->mtProgress, res); - next->stopReading = True; - next->stopWriting = True; - Event_Set(&next->canRead); - Event_Set(&next->canWrite); - return res; + return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD; } - if (stop) + + res = MtProgress_GetError(&mtc->mtProgress); + + size = 0; + inData = NULL; + finished = True; + + if (res == SZ_OK) + { + size = mtc->blockSize; + if (mtc->inStream) + { + if (!t->inBuf) + { + t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize); + if (!t->inBuf) + res = SZ_ERROR_MEM; + } + if (res == SZ_OK) + { + res = FullRead(mtc->inStream, t->inBuf, &size); + readProcessed = mtc->readProcessed + size; + mtc->readProcessed = readProcessed; + } + if (res != SZ_OK) + { + mtc->readRes = res; + /* after reading error - we can stop encoding of previous blocks */ + MtProgress_SetError(&mtc->mtProgress, res); + } + else + finished = (size != mtc->blockSize); + } + else + { + size_t rem; + readProcessed = mtc->readProcessed; + rem = mtc->inDataSize - (size_t)readProcessed; + if (size > rem) + size = rem; + inData = mtc->inData + (size_t)readProcessed; + readProcessed += size; + mtc->readProcessed = readProcessed; + finished = (mtc->inDataSize == (size_t)readProcessed); + } + } + + /* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */ + + res2 = SZ_OK; + + if (Semaphore_Wait(&mtc->blocksSemaphore) != 0) + { + res2 = SZ_ERROR_THREAD; + if (res == SZ_OK) + { + res = res2; + // MtProgress_SetError(&mtc->mtProgress, res); + } + } + + bi = mtc->blockIndex; + + if (++mtc->blockIndex >= mtc->numBlocksMax) + mtc->blockIndex = 0; + + bufIndex = (unsigned)(int)-1; + + if (res == SZ_OK) + res = MtProgress_GetError(&mtc->mtProgress); + + if (res != SZ_OK) + finished = True; + + if (!finished) + { + if (mtc->numStartedThreads < mtc->numStartedThreadsLimit + && mtc->expectedDataSize != readProcessed) + { + res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]); + if (res == SZ_OK) + mtc->numStartedThreads++; + else + { + MtProgress_SetError(&mtc->mtProgress, res); + finished = True; + } + } + } + + if (finished) + mtc->stopReading = True; + + RINOK_THREAD(Event_Set(&mtc->readEvent)) + + if (res2 != SZ_OK) + return res2; + + if (res == SZ_OK) + { + CriticalSection_Enter(&mtc->cs); + bufIndex = mtc->freeBlockHead; + mtc->freeBlockHead = mtc->freeBlockList[bufIndex]; + CriticalSection_Leave(&mtc->cs); + + res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, + mtc->inStream ? t->inBuf : inData, size, finished); + + // MtProgress_Reinit(&mtc->mtProgress, t->index); + + if (res != SZ_OK) + MtProgress_SetError(&mtc->mtProgress, res); + } + + { + CMtCoderBlock *block = &mtc->blocks[bi]; + block->res = res; + block->bufIndex = bufIndex; + block->finished = finished; + } + + #ifdef MTCODER__USE_WRITE_THREAD + RINOK_THREAD(Event_Set(&mtc->writeEvents[bi])) + #else + { + unsigned wi; + { + CriticalSection_Enter(&mtc->cs); + wi = mtc->writeIndex; + if (wi == bi) + mtc->writeIndex = (unsigned)(int)-1; + else + mtc->ReadyBlocks[bi] = True; + CriticalSection_Leave(&mtc->cs); + } + + if (wi != bi) + { + if (res != SZ_OK || finished) + return 0; + continue; + } + + if (mtc->writeRes != SZ_OK) + res = mtc->writeRes; + + for (;;) + { + if (res == SZ_OK && bufIndex != (unsigned)(int)-1) + { + res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex); + if (res != SZ_OK) + { + mtc->writeRes = res; + MtProgress_SetError(&mtc->mtProgress, res); + } + } + + if (++wi >= mtc->numBlocksMax) + wi = 0; + { + BoolInt isReady; + + CriticalSection_Enter(&mtc->cs); + + if (bufIndex != (unsigned)(int)-1) + { + mtc->freeBlockList[bufIndex] = mtc->freeBlockHead; + mtc->freeBlockHead = bufIndex; + } + + isReady = mtc->ReadyBlocks[wi]; + + if (isReady) + mtc->ReadyBlocks[wi] = False; + else + mtc->writeIndex = wi; + + CriticalSection_Leave(&mtc->cs); + + RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore)) + + if (!isReady) + break; + } + + { + CMtCoderBlock *block = &mtc->blocks[wi]; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + bufIndex = block->bufIndex; + finished = block->finished; + } + } + } + #endif + + if (finished || res != SZ_OK) return 0; } } -void MtCoder_Construct(CMtCoder* p) + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtCoderThread *t = (CMtCoderThread *)pp; + for (;;) + { + if (Event_Wait(&t->startEvent) != 0) + return SZ_ERROR_THREAD; + if (t->stop) + return 0; + { + SRes res = ThreadFunc2(t); + CMtCoder *mtc = t->mtCoder; + if (res != SZ_OK) + { + MtProgress_SetError(&mtc->mtProgress, res); + } + + #ifndef MTCODER__USE_WRITE_THREAD + { + unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); + if (numFinished == mtc->numStartedThreads) + if (Event_Set(&mtc->finishedEvent) != 0) + return SZ_ERROR_THREAD; + } + #endif + } + } +} + + + +void MtCoder_Construct(CMtCoder *p) { unsigned i; - p->alloc = 0; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + + p->blockSize = 0; + p->numThreadsMax = 0; + p->expectedDataSize = (UInt64)(Int64)-1; + + p->inStream = NULL; + p->inData = NULL; + p->inDataSize = 0; + + p->progress = NULL; + p->allocBig = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + Event_Construct(&p->readEvent); + Semaphore_Construct(&p->blocksSemaphore); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) { - CMtThread *t = &p->threads[i]; + CMtCoderThread *t = &p->threads[i]; + t->mtCoder = p; t->index = i; - CMtThread_Construct(t, p); + t->inBuf = NULL; + t->stop = False; + Event_Construct(&t->startEvent); + Thread_Construct(&t->thread); } + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Construct(&p->writeEvents[i]); + #else + Event_Construct(&p->finishedEvent); + #endif + CriticalSection_Init(&p->cs); CriticalSection_Init(&p->mtProgress.cs); } -void MtCoder_Destruct(CMtCoder* p) + + + +static void MtCoder_Free(CMtCoder *p) { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - CMtThread_Destruct(&p->threads[i]); + + /* + p->stopReading = True; + if (Event_IsCreated(&p->readEvent)) + Event_Set(&p->readEvent); + */ + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + MtCoderThread_Destruct(&p->threads[i]); + + Event_Close(&p->readEvent); + Semaphore_Close(&p->blocksSemaphore); + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Close(&p->writeEvents[i]); + #else + Event_Close(&p->finishedEvent); + #endif +} + + +void MtCoder_Destruct(CMtCoder *p) +{ + MtCoder_Free(p); + CriticalSection_Delete(&p->cs); CriticalSection_Delete(&p->mtProgress.cs); } + SRes MtCoder_Code(CMtCoder *p) { - unsigned i, numThreads = p->numThreads; + unsigned numThreads = p->numThreadsMax; + unsigned numBlocksMax; + unsigned i; SRes res = SZ_OK; - p->res = SZ_OK; + + if (numThreads > MTCODER__THREADS_MAX) + numThreads = MTCODER__THREADS_MAX; + numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads); + + if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; + + if (numBlocksMax > MTCODER__BLOCKS_MAX) + numBlocksMax = MTCODER__BLOCKS_MAX; + + if (p->blockSize != p->allocatedBufsSize) + { + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CMtCoderThread *t = &p->threads[i]; + if (t->inBuf) + { + ISzAlloc_Free(p->allocBig, t->inBuf); + t->inBuf = NULL; + } + } + p->allocatedBufsSize = p->blockSize; + } + + p->readRes = SZ_OK; MtProgress_Init(&p->mtProgress, p->progress); - for (i = 0; i < numThreads; i++) + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < numBlocksMax; i++) + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i])); + } + #else + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + #endif + { - RINOK(CMtThread_Prepare(&p->threads[i])); + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); + + if (Semaphore_IsCreated(&p->blocksSemaphore)) + { + RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore)); + } + RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); } - for (i = 0; i < numThreads; i++) + for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) + p->freeBlockList[i] = i + 1; + p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1; + p->freeBlockHead = 0; + + p->readProcessed = 0; + p->blockIndex = 0; + p->numBlocksMax = numBlocksMax; + p->stopReading = False; + + #ifndef MTCODER__USE_WRITE_THREAD + p->writeIndex = 0; + p->writeRes = SZ_OK; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->ReadyBlocks[i] = False; + p->numFinishedThreads = 0; + #endif + + p->numStartedThreadsLimit = numThreads; + p->numStartedThreads = 0; + + // for (i = 0; i < numThreads; i++) { - CMtThread *t = &p->threads[i]; - CLoopThread *lt = &t->thread; + CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++]; + RINOK(MtCoderThread_CreateAndStart(nextThread)); + } - if (!Thread_WasCreated(<->thread)) + RINOK_THREAD(Event_Set(&p->readEvent)) + + #ifdef MTCODER__USE_WRITE_THREAD + { + unsigned bi = 0; + + for (;; bi++) { - lt->func = ThreadFunc; - lt->param = t; + if (bi >= numBlocksMax) + bi = 0; + + RINOK_THREAD(Event_Wait(&p->writeEvents[bi])) - if (LoopThread_Create(lt) != SZ_OK) { - res = SZ_ERROR_THREAD; - break; + const CMtCoderBlock *block = &p->blocks[bi]; + unsigned bufIndex = block->bufIndex; + BoolInt finished = block->finished; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + + if (bufIndex != (unsigned)(int)-1) + { + if (res == SZ_OK) + { + res = p->mtCallback->Write(p->mtCallbackObject, bufIndex); + if (res != SZ_OK) + MtProgress_SetError(&p->mtProgress, res); + } + + CriticalSection_Enter(&p->cs); + { + p->freeBlockList[bufIndex] = p->freeBlockHead; + p->freeBlockHead = bufIndex; + } + CriticalSection_Leave(&p->cs); + } + + RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore)) + + if (finished) + break; } } } + #else + { + WRes wres = Event_Wait(&p->finishedEvent); + res = MY_SRes_HRESULT_FROM_WRes(wres); + } + #endif if (res == SZ_OK) - { - unsigned j; - for (i = 0; i < numThreads; i++) - { - CMtThread *t = &p->threads[i]; - if (LoopThread_StartSubThread(&t->thread) != SZ_OK) - { - res = SZ_ERROR_THREAD; - p->threads[0].stopReading = True; - break; - } - } + res = p->readRes; - Event_Set(&p->threads[0].canWrite); - Event_Set(&p->threads[0].canRead); + if (res == SZ_OK) + res = p->mtProgress.res; - for (j = 0; j < i; j++) - LoopThread_WaitSubThread(&p->threads[j].thread); - } + #ifndef MTCODER__USE_WRITE_THREAD + if (res == SZ_OK) + res = p->writeRes; + #endif - for (i = 0; i < numThreads; i++) - CMtThread_CloseEvents(&p->threads[i]); - return (res == SZ_OK) ? p->res : res; + if (res != SZ_OK) + MtCoder_Free(p); + return res; } + +#endif diff --git a/src/third-party/lzma/MtCoder.h b/src/third-party/lzma/MtCoder.h index 705208ed3..603329d36 100644 --- a/src/third-party/lzma/MtCoder.h +++ b/src/third-party/lzma/MtCoder.h @@ -1,98 +1,141 @@ /* MtCoder.h -- Multi-thread Coder -2009-11-19 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN -typedef struct -{ - CThread thread; - CAutoResetEvent startEvent; - CAutoResetEvent finishedEvent; - int stop; - - THREAD_FUNC_TYPE func; - LPVOID param; - THREAD_FUNC_RET_TYPE res; -} CLoopThread; - -void LoopThread_Construct(CLoopThread *p); -void LoopThread_Close(CLoopThread *p); -WRes LoopThread_Create(CLoopThread *p); -WRes LoopThread_StopAndWait(CLoopThread *p); -WRes LoopThread_StartSubThread(CLoopThread *p); -WRes LoopThread_WaitSubThread(CLoopThread *p); +/* + if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream + if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream +*/ +/* #define MTCODER__USE_WRITE_THREAD */ #ifndef _7ZIP_ST -#define NUM_MT_CODER_THREADS_MAX 32 + #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) + #define MTCODER__THREADS_MAX 64 + #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3) #else -#define NUM_MT_CODER_THREADS_MAX 1 + #define MTCODER__THREADS_MAX 1 + #define MTCODER__BLOCKS_MAX 1 #endif + +#ifndef _7ZIP_ST + + typedef struct { - UInt64 totalInSize; - UInt64 totalOutSize; - ICompressProgress *progress; - SRes res; - CCriticalSection cs; - UInt64 inSizes[NUM_MT_CODER_THREADS_MAX]; - UInt64 outSizes[NUM_MT_CODER_THREADS_MAX]; -} CMtProgress; + ICompressProgress vt; + CMtProgress *mtProgress; + UInt64 inSize; + UInt64 outSize; +} CMtProgressThunk; + +void MtProgressThunk_CreateVTable(CMtProgressThunk *p); + +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize); struct _CMtCoder; + typedef struct { struct _CMtCoder *mtCoder; - Byte *outBuf; - size_t outBufSize; - Byte *inBuf; - size_t inBufSize; unsigned index; - CLoopThread thread; + int stop; + Byte *inBuf; + + CAutoResetEvent startEvent; + CThread thread; +} CMtCoderThread; - Bool stopReading; - Bool stopWriting; - CAutoResetEvent canRead; - CAutoResetEvent canWrite; -} CMtThread; typedef struct { - SRes (*Code)(void *p, unsigned index, Byte *dest, size_t *destSize, + SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished); -} IMtCoderCallback; + SRes (*Write)(void *p, unsigned outBufIndex); +} IMtCoderCallback2; + + +typedef struct +{ + SRes res; + unsigned bufIndex; + BoolInt finished; +} CMtCoderBlock; + typedef struct _CMtCoder { - size_t blockSize; - size_t destBlockSize; - unsigned numThreads; + /* input variables */ - ISeqInStream *inStream; - ISeqOutStream *outStream; - ICompressProgress *progress; - ISzAlloc *alloc; + size_t blockSize; /* size of input block */ + unsigned numThreadsMax; + UInt64 expectedDataSize; + + ISeqInStream *inStream; + const Byte *inData; + size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr allocBig; + + IMtCoderCallback2 *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + CAutoResetEvent readEvent; + CSemaphore blocksSemaphore; + + BoolInt stopReading; + SRes readRes; + + #ifdef MTCODER__USE_WRITE_THREAD + CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX]; + #else + CAutoResetEvent finishedEvent; + SRes writeRes; + unsigned writeIndex; + Byte ReadyBlocks[MTCODER__BLOCKS_MAX]; + LONG numFinishedThreads; + #endif + + unsigned numStartedThreadsLimit; + unsigned numStartedThreads; + + unsigned numBlocksMax; + unsigned blockIndex; + UInt64 readProcessed; - IMtCoderCallback *mtCallback; CCriticalSection cs; - SRes res; + + unsigned freeBlockHead; + unsigned freeBlockList[MTCODER__BLOCKS_MAX]; CMtProgress mtProgress; - CMtThread threads[NUM_MT_CODER_THREADS_MAX]; + CMtCoderBlock blocks[MTCODER__BLOCKS_MAX]; + CMtCoderThread threads[MTCODER__THREADS_MAX]; } CMtCoder; -void MtCoder_Construct(CMtCoder* p); -void MtCoder_Destruct(CMtCoder* p); + +void MtCoder_Construct(CMtCoder *p); +void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); + +#endif + + EXTERN_C_END #endif diff --git a/src/third-party/lzma/MtDec.c b/src/third-party/lzma/MtDec.c new file mode 100644 index 000000000..25a8b046d --- /dev/null +++ b/src/third-party/lzma/MtDec.c @@ -0,0 +1,1138 @@ +/* MtDec.c -- Multi-thread Decoder +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + +struct __CMtDecBufLink +{ + struct __CMtDecBufLink *next; + void *pad[3]; +}; + +typedef struct __CMtDecBufLink CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + BoolInt wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + BoolInt needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + + BoolInt canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + // PRF(printf("\nParse size = %d\n", (unsigned)size)) + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + p->crossStart = 0; + p->crossEnd = 0; + + if (crossSize != 0) + memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data + afterEndData_Size = size - parse.srcSize; + afterEndData = parseData + parse.srcSize; + + // we reduce data size to required bytes (parsed only) + inDataSize -= (size - parse.srcSize); + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + BoolInt isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + BoolInt isErrorMode = False; + BoolInt canRecode = True; + BoolInt needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, + &needContinue, + &canRecode); + + // res= E_INVALIDARG; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return res; +} + +static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtDecThread *t = (CMtDecThread *)pp; + + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + #ifdef USE_ALLOCA + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + WRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return E_FAIL; + } +} + +#endif diff --git a/src/third-party/lzma/MtDec.h b/src/third-party/lzma/MtDec.h new file mode 100644 index 000000000..9864cc874 --- /dev/null +++ b/src/third-party/lzma/MtDec.h @@ -0,0 +1,201 @@ +/* MtDec.h -- Multi-thread Decoder +2018-07-04 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + BoolInt canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode); +} IMtDecCallback; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + BoolInt exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + BoolInt isAllocError; + BoolInt overflow; + SRes threadingErrorSRes; + + BoolInt needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + BoolInt wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + BoolInt readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + BoolInt needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif diff --git a/src/third-party/lzma/Ppmd.h b/src/third-party/lzma/Ppmd.h index 25ce2d28f..4b9941521 100644 --- a/src/third-party/lzma/Ppmd.h +++ b/src/third-party/lzma/Ppmd.h @@ -1,5 +1,5 @@ /* Ppmd.h -- PPMD codec common code -2013-01-18 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #ifndef __PPMD_H @@ -77,8 +77,8 @@ typedef CPpmd_Byte_Ref; #define PPMD_SetAllBitsIn256Bytes(p) \ - { unsigned i; for (i = 0; i < 256 / sizeof(p[0]); i += 8) { \ - p[i+7] = p[i+6] = p[i+5] = p[i+4] = p[i+3] = p[i+2] = p[i+1] = p[i+0] = ~(size_t)0; }} + { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ + p[z+7] = p[z+6] = p[z+5] = p[z+4] = p[z+3] = p[z+2] = p[z+1] = p[z+0] = ~(size_t)0; }} EXTERN_C_END diff --git a/src/third-party/lzma/Ppmd7.c b/src/third-party/lzma/Ppmd7.c index 735651f5d..80e7de9a6 100644 --- a/src/third-party/lzma/Ppmd7.c +++ b/src/third-party/lzma/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2015-09-28 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -15,7 +15,7 @@ static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x #define UNIT_SIZE 12 #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) -#define U2I(nu) (p->Units2Indx[(nu) - 1]) +#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) #define I2U(indx) (p->Indx2Units[indx]) #ifdef PPMD_32BIT @@ -88,29 +88,31 @@ void Ppmd7_Construct(CPpmd7 *p) memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); } -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc) +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->Base); + ISzAlloc_Free(alloc, p->Base); p->Size = 0; p->Base = 0; } -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { - if (p->Base == 0 || p->Size != size) + if (!p->Base || p->Size != size) { + size_t size2; Ppmd7_Free(p, alloc); + size2 = 0 + #ifndef PPMD_32BIT + + UNIT_SIZE + #endif + ; p->AlignOffset = #ifdef PPMD_32BIT (4 - size) & 3; #else 4 - (size & 3); #endif - if ((p->Base = (Byte *)alloc->Alloc(alloc, p->AlignOffset + size - #ifndef PPMD_32BIT - + UNIT_SIZE - #endif - )) == 0) + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size + size2)) == 0) return False; p->Size = size; } @@ -340,7 +342,7 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) +static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) { CPpmd_State upState; CTX_PTR c = p->MinContext; @@ -513,7 +515,7 @@ static void UpdateModel(CPpmd7 *p) /* Expand for one UNIT */ unsigned oldNU = ns1 >> 1; unsigned i = U2I(oldNU); - if (i != U2I(oldNU + 1)) + if (i != U2I((size_t)oldNU + 1)) { void *ptr = AllocUnits(p, i + 1); void *oldPtr; @@ -639,10 +641,10 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) unsigned nonMasked = p->MinContext->NumStats - numMasked; if (p->MinContext->NumStats != 256) { - see = p->See[p->NS2Indx[nonMasked - 1]] + + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + - 2 * (p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + - 4 * (numMasked > nonMasked) + + 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + + 4 * (unsigned)(numMasked > nonMasked) + p->HiBitsFlag; { unsigned r = (see->Summ >> see->Shift); diff --git a/src/third-party/lzma/Ppmd7.h b/src/third-party/lzma/Ppmd7.h index 56e81eb12..cce93f120 100644 --- a/src/third-party/lzma/Ppmd7.h +++ b/src/third-party/lzma/Ppmd7.h @@ -1,5 +1,5 @@ /* Ppmd7.h -- PPMdH compression codec -2010-03-12 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ /* This code supports virtual RangeDecoder and includes the implementation @@ -60,8 +60,8 @@ typedef struct } CPpmd7; void Ppmd7_Construct(CPpmd7 *p); -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc); -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) @@ -86,10 +86,10 @@ void Ppmd7_Update2(CPpmd7 *p); void Ppmd7_UpdateBin(CPpmd7 *p); #define Ppmd7_GetBinSumm(p) \ - &p->BinSumm[Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ - p->NS2BSIndx[Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ + &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ + p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ - 2 * p->HB2Flag[Ppmd7Context_OneState(p->MinContext)->Symbol] + \ + 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ ((p->RunLength >> 26) & 0x20)] CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); @@ -97,26 +97,28 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); /* ---------- Decode ---------- */ -typedef struct +typedef struct IPpmd7_RangeDec IPpmd7_RangeDec; + +struct IPpmd7_RangeDec { - UInt32 (*GetThreshold)(void *p, UInt32 total); - void (*Decode)(void *p, UInt32 start, UInt32 size); - UInt32 (*DecodeBit)(void *p, UInt32 size0); -} IPpmd7_RangeDec; + UInt32 (*GetThreshold)(const IPpmd7_RangeDec *p, UInt32 total); + void (*Decode)(const IPpmd7_RangeDec *p, UInt32 start, UInt32 size); + UInt32 (*DecodeBit)(const IPpmd7_RangeDec *p, UInt32 size0); +}; typedef struct { - IPpmd7_RangeDec p; + IPpmd7_RangeDec vt; UInt32 Range; UInt32 Code; IByteIn *Stream; } CPpmd7z_RangeDec; void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); #define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc); +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); /* ---------- Encode ---------- */ diff --git a/src/third-party/lzma/Ppmd7Dec.c b/src/third-party/lzma/Ppmd7Dec.c index 3d01d7644..202640710 100644 --- a/src/third-party/lzma/Ppmd7Dec.c +++ b/src/third-party/lzma/Ppmd7Dec.c @@ -1,5 +1,5 @@ /* Ppmd7Dec.c -- PPMdH Decoder -2010-03-12 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,49 +8,51 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) { unsigned i; p->Code = 0; p->Range = 0xFFFFFFFF; - if (p->Stream->Read((void *)p->Stream) != 0) + if (IByteIn_Read(p->Stream) != 0) return False; for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); return (p->Code < 0xFFFFFFFF); } -static UInt32 Range_GetThreshold(void *pp, UInt32 total) +#define GET_Ppmd7z_RangeDec CPpmd7z_RangeDec *p = CONTAINER_FROM_VTBL(pp, CPpmd7z_RangeDec, vt); + +static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; - return (p->Code) / (p->Range /= total); + GET_Ppmd7z_RangeDec + return p->Code / (p->Range /= total); } static void Range_Normalize(CPpmd7z_RangeDec *p) { if (p->Range < kTopValue) { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); p->Range <<= 8; if (p->Range < kTopValue) { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); p->Range <<= 8; } } } -static void Range_Decode(void *pp, UInt32 start, UInt32 size) +static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; + GET_Ppmd7z_RangeDec p->Code -= start * p->Range; p->Range *= size; Range_Normalize(p); } -static UInt32 Range_DecodeBit(void *pp, UInt32 size0) +static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; + GET_Ppmd7z_RangeDec UInt32 newBound = (p->Range >> 14) * size0; UInt32 symbol; if (p->Code < newBound) @@ -70,15 +72,15 @@ static UInt32 Range_DecodeBit(void *pp, UInt32 size0) void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) { - p->p.GetThreshold = Range_GetThreshold; - p->p.Decode = Range_Decode; - p->p.DecodeBit = Range_DecodeBit; + p->vt.GetThreshold = Range_GetThreshold; + p->vt.Decode = Range_Decode; + p->vt.DecodeBit = Range_DecodeBit; } #define MASK(sym) ((signed char *)charMask)[sym] -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc) +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc) { size_t charMask[256 / sizeof(size_t)]; if (p->MinContext->NumStats != 1) diff --git a/src/third-party/lzma/Ppmd7Enc.c b/src/third-party/lzma/Ppmd7Enc.c index 9b49e5d44..a74d3002b 100644 --- a/src/third-party/lzma/Ppmd7Enc.c +++ b/src/third-party/lzma/Ppmd7Enc.c @@ -1,5 +1,5 @@ /* Ppmd7Enc.c -- PPMdH Encoder -2015-09-28 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -23,7 +23,7 @@ static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) Byte temp = p->Cache; do { - p->Stream->Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + IByteOut_Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); temp = 0xFF; } while (--p->CacheSize != 0); diff --git a/src/third-party/lzma/Sha256.c b/src/third-party/lzma/Sha256.c index 47e2f42b9..90994e5ab 100644 --- a/src/third-party/lzma/Sha256.c +++ b/src/third-party/lzma/Sha256.c @@ -1,5 +1,5 @@ /* Crypto/Sha256.c -- SHA-256 Hash -2015-11-14 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -45,7 +45,7 @@ void Sha256_Init(CSha256 *p) #ifdef _SHA256_UNROLL2 #define R(a,b,c,d,e,f,g,h, i) \ - h += S1(e) + Ch(e,f,g) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ d += h; \ h += S0(a) + Maj(a, b, c) @@ -73,7 +73,7 @@ void Sha256_Init(CSha256 *p) #define h(i) T[(7-(i))&7] #define R(i) \ - h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ + h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \ diff --git a/src/third-party/lzma/Threads.c b/src/third-party/lzma/Threads.c index ece07e618..8fd86f224 100644 --- a/src/third-party/lzma/Threads.c +++ b/src/third-party/lzma/Threads.c @@ -1,5 +1,5 @@ /* Threads.c -- multithreading library -2014-09-21 : Igor Pavlov : Public domain */ +2017-06-26 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,18 +12,20 @@ static WRes GetError() { DWORD res = GetLastError(); - return (res) ? (WRes)(res) : 1; + return res ? (WRes)res : 1; } -WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); } -WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } +static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); } +static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } WRes HandlePtr_Close(HANDLE *p) { if (*p != NULL) + { if (!CloseHandle(*p)) return GetError(); - *p = NULL; + *p = NULL; + } return 0; } @@ -49,7 +51,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) return HandleToWRes(*p); } -WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) +static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) { *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL); return HandleToWRes(*p); diff --git a/src/third-party/lzma/Threads.h b/src/third-party/lzma/Threads.h index e927208d7..f913241ae 100644 --- a/src/third-party/lzma/Threads.h +++ b/src/third-party/lzma/Threads.h @@ -1,5 +1,5 @@ /* Threads.h -- multithreading library -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-18 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H @@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); typedef HANDLE CSemaphore; -#define Semaphore_Construct(p) (*p) = NULL +#define Semaphore_Construct(p) *(p) = NULL +#define Semaphore_IsCreated(p) (*(p) != NULL) #define Semaphore_Close(p) HandlePtr_Close(p) #define Semaphore_Wait(p) Handle_WaitObject(*(p)) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); diff --git a/src/third-party/lzma/Util/7z/7zMain.c b/src/third-party/lzma/Util/7z/7zMain.c index 00c6dbb6b..1c02b48ec 100644 --- a/src/third-party/lzma/Util/7z/7zMain.c +++ b/src/third-party/lzma/Util/7z/7zMain.c @@ -1,11 +1,13 @@ /* 7zMain.c - Test application for 7z Decoder -2015-08-02 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" #include #include +#include "../../CpuArch.h" + #include "../../7z.h" #include "../../7zAlloc.h" #include "../../7zBuf.h" @@ -23,7 +25,17 @@ #endif #endif -static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +#define kInputBufSize ((size_t)1 << 18) + +static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + + +static void Print(const char *s) +{ + fputs(s, stdout); +} + static int Buf_EnsureSize(CBuf *dest, size_t size) { @@ -164,7 +176,7 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s char defaultChar = '_'; BOOL defUsed; unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed); + numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0; @@ -190,7 +202,7 @@ static WRes MyCreateDir(const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return CreateDirectoryW(name, NULL) ? 0 : GetLastError(); + return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); #else @@ -215,7 +227,7 @@ static WRes MyCreateDir(const UInt16 *name) static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, name); + return OutFile_OpenW(p, (LPCWSTR)name); #else CBuf buf; WRes res; @@ -227,6 +239,7 @@ static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) #endif } + static SRes PrintString(const UInt16 *s) { CBuf buf; @@ -238,12 +251,12 @@ static SRes PrintString(const UInt16 *s) #endif ); if (res == SZ_OK) - fputs((const char *)buf.data, stdout); + Print((const char *)buf.data); Buf_Free(&buf, &g_Alloc); return res; } -static void UInt64ToStr(UInt64 value, char *s) +static void UInt64ToStr(UInt64 value, char *s, int numDigits) { char temp[32]; int pos = 0; @@ -253,6 +266,10 @@ static void UInt64ToStr(UInt64 value, char *s) value /= 10; } while (value != 0); + + for (numDigits -= pos; numDigits > 0; numDigits--) + *s++ = ' '; + do *s++ = temp[--pos]; while (pos); @@ -266,8 +283,10 @@ static char *UIntToStr(char *s, unsigned value, int numDigits) do temp[pos++] = (char)('0' + (value % 10)); while (value /= 10); + for (numDigits -= pos; numDigits > 0; numDigits--) *s++ = '0'; + do *s++ = temp[--pos]; while (pos); @@ -310,10 +329,10 @@ static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) ms[1] = 29; for (mon = 0;; mon++) { - unsigned s = ms[mon]; - if (v < s) + unsigned d = ms[mon]; + if (v < d) break; - v -= s; + v -= d; } s = UIntToStr(s, year, 4); *s++ = '-'; UIntToStr_2(s, mon + 1); s[2] = '-'; s += 3; @@ -323,12 +342,19 @@ static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) UIntToStr_2(s, sec); s[2] = 0; } -void PrintError(char *sz) +static void PrintLF() { - printf("\nERROR: %s\n", sz); + Print("\n"); } -static void GetAttribString(UInt32 wa, Bool isDir, char *s) +static void PrintError(char *s) +{ + Print("\nERROR: "); + Print(s); + PrintLF(); +} + +static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); @@ -343,25 +369,27 @@ static void GetAttribString(UInt32 wa, Bool isDir, char *s) #endif } + // #define NUM_PARENTS_MAX 128 int MY_CDECL main(int numargs, char *args[]) { - CFileInStream archiveStream; - CLookToRead lookStream; - CSzArEx db; - SRes res; ISzAlloc allocImp; ISzAlloc allocTempImp; + + CFileInStream archiveStream; + CLookToRead2 lookStream; + CSzArEx db; + SRes res; UInt16 *temp = NULL; size_t tempSize = 0; // UInt32 parents[NUM_PARENTS_MAX]; - printf("\n7z ANSI-C Decoder " MY_VERSION_COPYRIGHT_DATE "\n\n"); + Print("\n7z Decoder " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"); if (numargs == 1) { - printf( + Print( "Usage: 7zDec \n\n" "\n" " e: Extract files from archive (without using directory names)\n" @@ -370,7 +398,7 @@ int MY_CDECL main(int numargs, char *args[]) " x: eXtract files with full paths\n"); return 0; } - + if (numargs < 3) { PrintError("incorrect command"); @@ -381,11 +409,9 @@ int MY_CDECL main(int numargs, char *args[]) g_FileCodePage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; #endif - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; + allocImp = g_Alloc; + allocTempImp = g_Alloc; #ifdef UNDER_CE if (InFile_OpenW(&archiveStream.file, L"\test.7z")) @@ -398,16 +424,31 @@ int MY_CDECL main(int numargs, char *args[]) } FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); - - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; + res = SZ_OK; + + { + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } + } + CrcGenerateTable(); - + SzArEx_Init(&db); - - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + + if (res == SZ_OK) + { + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); + } if (res == SZ_OK) { @@ -477,7 +518,7 @@ int MY_CDECL main(int numargs, char *args[]) GetAttribString(SzBitWithVals_Check(&db.Attribs, i) ? db.Attribs.Vals[i] : 0, isDir, attr); fileSize = SzArEx_GetFileSize(&db, i); - UInt64ToStr(fileSize, s); + UInt64ToStr(fileSize, s, 10); if (SzBitWithVals_Check(&db.MTime, i)) ConvertFileTimeToString(&db.MTime.Vals[i], t); @@ -489,29 +530,33 @@ int MY_CDECL main(int numargs, char *args[]) t[j] = '\0'; } - printf("%s %s %10s ", t, attr, s); + Print(t); + Print(" "); + Print(attr); + Print(" "); + Print(s); + Print(" "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); - printf("\n"); + Print("/"); + PrintLF(); continue; } - fputs(testCommand ? + Print(testCommand ? "Testing ": - "Extracting ", - stdout); + "Extracting "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); + Print("/"); else { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp); @@ -543,7 +588,7 @@ int MY_CDECL main(int numargs, char *args[]) if (isDir) { MyCreateDir(destPath); - printf("\n"); + PrintLF(); continue; } else if (OutFile_OpenUtf16(&outFile, destPath)) @@ -561,6 +606,31 @@ int MY_CDECL main(int numargs, char *args[]) res = SZ_ERROR_FAIL; break; } + + #ifdef USE_WINDOWS_FILE + { + FILETIME mtime, ctime; + FILETIME *mtimePtr = NULL; + FILETIME *ctimePtr = NULL; + + if (SzBitWithVals_Check(&db.MTime, i)) + { + const CNtfsFileTime *t = &db.MTime.Vals[i]; + mtime.dwLowDateTime = (DWORD)(t->Low); + mtime.dwHighDateTime = (DWORD)(t->High); + mtimePtr = &mtime; + } + if (SzBitWithVals_Check(&db.CTime, i)) + { + const CNtfsFileTime *t = &db.CTime.Vals[i]; + ctime.dwLowDateTime = (DWORD)(t->Low); + ctime.dwHighDateTime = (DWORD)(t->High); + ctimePtr = &ctime; + } + if (mtimePtr || ctimePtr) + SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); + } + #endif if (File_Close(&outFile)) { @@ -571,23 +641,31 @@ int MY_CDECL main(int numargs, char *args[]) #ifdef USE_WINDOWS_FILE if (SzBitWithVals_Check(&db.Attribs, i)) - SetFileAttributesW(destPath, db.Attribs.Vals[i]); + { + UInt32 attrib = db.Attribs.Vals[i]; + /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. + We remove posix bits, if we detect posix mode field */ + if ((attrib & 0xF0000000) != 0) + attrib &= 0x7FFF; + SetFileAttributesW((LPCWSTR)destPath, attrib); + } #endif } - printf("\n"); + PrintLF(); } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } } - SzArEx_Free(&db, &allocImp); SzFree(NULL, temp); + SzArEx_Free(&db, &allocImp); + ISzAlloc_Free(&allocImp, lookStream.buf); File_Close(&archiveStream.file); if (res == SZ_OK) { - printf("\nEverything is Ok\n"); + Print("\nEverything is Ok\n"); return 0; } @@ -598,7 +676,11 @@ int MY_CDECL main(int numargs, char *args[]) else if (res == SZ_ERROR_CRC) PrintError("CRC error"); else - printf("\nERROR #%d\n", res); + { + char s[32]; + UInt64ToStr(res, s, 0); + PrintError(s); + } return 1; } diff --git a/src/third-party/lzma/Util/Lzma/LzmaUtil.c b/src/third-party/lzma/Util/Lzma/LzmaUtil.c index 98331b49d..82130e85d 100644 --- a/src/third-party/lzma/Util/Lzma/LzmaUtil.c +++ b/src/third-party/lzma/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -7,26 +7,29 @@ #include #include +#include "../../CpuArch.h" + #include "../../Alloc.h" #include "../../7zFile.h" #include "../../7zVersion.h" #include "../../LzmaDec.h" #include "../../LzmaEnc.h" -const char *kCantReadMessage = "Can not read input file"; -const char *kCantWriteMessage = "Can not write output file"; -const char *kCantAllocateMessage = "Can not allocate memory"; -const char *kDataErrorMessage = "Data error"; +static const char * const kCantReadMessage = "Can not read input file"; +static const char * const kCantWriteMessage = "Can not write output file"; +static const char * const kCantAllocateMessage = "Can not allocate memory"; +static const char * const kDataErrorMessage = "Data error"; -void PrintHelp(char *buffer) +static void PrintHelp(char *buffer) { - strcat(buffer, "\nLZMA Utility " MY_VERSION_COPYRIGHT_DATE "\n" - "\nUsage: lzma inputFile outputFile\n" - " e: encode file\n" - " d: decode file\n"); + strcat(buffer, + "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" + "Usage: lzma inputFile outputFile\n" + " e: encode file\n" + " d: decode file\n"); } -int PrintError(char *buffer, const char *message) +static int PrintError(char *buffer, const char *message) { strcat(buffer, "\nError: "); strcat(buffer, message); @@ -34,20 +37,22 @@ int PrintError(char *buffer, const char *message) return 1; } -int PrintErrorNumber(char *buffer, SRes val) +static int PrintErrorNumber(char *buffer, SRes val) { sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); return 1; } -int PrintUserError(char *buffer) +static int PrintUserError(char *buffer) { return PrintError(buffer, "Incorrect command"); } + #define IN_BUF_SIZE (1 << 16) #define OUT_BUF_SIZE (1 << 16) + static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 unpackSize) { @@ -101,6 +106,7 @@ static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inS } } + static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream) { UInt64 unpackSize; @@ -163,14 +169,15 @@ static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 file return res; } -int main2(int numArgs, const char *args[], char *rs) + +static int main2(int numArgs, const char *args[], char *rs) { CFileSeqInStream inStream; CFileOutStream outStream; char c; int res; int encodeMode; - Bool useOutFile = False; + BoolInt useOutFile = False; FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); @@ -215,11 +222,11 @@ int main2(int numArgs, const char *args[], char *rs) { UInt64 fileSize; File_GetLength(&inStream.file, &fileSize); - res = Encode(&outStream.s, &inStream.s, fileSize, rs); + res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); } else { - res = Decode(&outStream.s, useOutFile ? &inStream.s : NULL); + res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL); } if (useOutFile) @@ -241,6 +248,7 @@ int main2(int numArgs, const char *args[], char *rs) return 0; } + int MY_CDECL main(int numArgs, const char *args[]) { char rs[800] = { 0 }; diff --git a/src/third-party/lzma/Util/Lzma/LzmaUtil.dsp b/src/third-party/lzma/Util/Lzma/LzmaUtil.dsp index df1f0cb00..eedde07d8 100644 --- a/src/third-party/lzma/Util/Lzma/LzmaUtil.dsp +++ b/src/third-party/lzma/Util/Lzma/LzmaUtil.dsp @@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\7zVersion.h # End Source File # Begin Source File @@ -160,9 +164,5 @@ SOURCE=..\..\Threads.c SOURCE=..\..\Threads.h # End Source File -# Begin Source File - -SOURCE=..\..\Types.h -# End Source File # End Target # End Project diff --git a/src/third-party/lzma/Util/SfxSetup/SfxSetup.c b/src/third-party/lzma/Util/SfxSetup/SfxSetup.c index b7ad5caf5..394369abd 100644 --- a/src/third-party/lzma/Util/SfxSetup/SfxSetup.c +++ b/src/third-party/lzma/Util/SfxSetup/SfxSetup.c @@ -1,5 +1,5 @@ /* SfxSetup.c - 7z SFX Setup -2015-11-08 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -20,9 +20,12 @@ #include "../../7zCrc.h" #include "../../7zFile.h" #include "../../CpuArch.h" +#include "../../DllSecur.h" #define k_EXE_ExtIndex 2 +#define kInputBufSize ((size_t)1 << 18) + static const char * const kExts[] = { "bat" @@ -124,7 +127,7 @@ static WRes MyCreateDir(const WCHAR *name) #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufferSize]; size_t numPrevBytes = 0; @@ -160,7 +163,7 @@ static Bool FindSignature(CSzFile *stream, UInt64 *resPos) } } -static Bool DoesFileOrDirExist(const WCHAR *path) +static BoolInt DoesFileOrDirExist(const WCHAR *path) { WIN32_FIND_DATAW fd; HANDLE handle; @@ -237,7 +240,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, #endif { CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; SRes res = SZ_OK; ISzAlloc allocImp; @@ -251,9 +254,11 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, DWORD winRes; const wchar_t *cmdLineParams; const char *errorMessage = NULL; - Bool useShellExecute = True; + BoolInt useShellExecute = True; DWORD exitCode = 0; + LoadSecurityDlls(); + #ifdef _CONSOLE SetConsoleCtrlHandler(HandlerRoutine, TRUE); #else @@ -272,7 +277,8 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, allocTempImp.Free = SzFreeTemp; FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; winRes = GetModuleFileNameW(NULL, sfxPath, MAX_PATH); if (winRes == 0 || winRes > MAX_PATH) @@ -281,7 +287,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, cmdLineParams = GetCommandLineW(); #ifndef UNDER_CE { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; cmdLineParams++) { wchar_t c = *cmdLineParams; @@ -373,14 +379,22 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (res == SZ_OK) { - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } } SzArEx_Init(&db); + if (res == SZ_OK) { - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); } if (res == SZ_OK) @@ -396,11 +410,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { size_t offset = 0; size_t outSizeProcessed = 0; - size_t len; WCHAR *temp; - len = SzArEx_GetFileNameUtf16(&db, i, NULL); - - if (len >= MAX_PATH) + + if (SzArEx_GetFileNameUtf16(&db, i, NULL) >= MAX_PATH) { res = SZ_ERROR_FAIL; break; @@ -408,9 +420,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp); @@ -515,16 +527,19 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { WCHAR *temp = path + pathLen; UInt32 j; - SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp); + SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp); for (j = 0; temp[j] != 0; j++) if (temp[j] == '/') temp[j] = CHAR_PATH_SEPARATOR; } } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } + SzArEx_Free(&db, &allocImp); + ISzAlloc_Free(&allocImp, lookStream.buf); + File_Close(&archiveStream.file); if (res == SZ_OK) diff --git a/src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp b/src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp index 931c537ba..be9de6dec 100644 --- a/src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp +++ b/src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp @@ -187,6 +187,14 @@ SOURCE=..\..\Delta.h # End Source File # Begin Source File +SOURCE=..\..\DllSecur.c +# End Source File +# Begin Source File + +SOURCE=..\..\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\Lzma2Dec.c # End Source File # Begin Source File diff --git a/src/third-party/lzma/Util/SfxSetup/makefile b/src/third-party/lzma/Util/SfxSetup/makefile index 490b51d9e..c9f59ccd3 100644 --- a/src/third-party/lzma/Util/SfxSetup/makefile +++ b/src/third-party/lzma/Util/SfxSetup/makefile @@ -1,4 +1,5 @@ PROG = 7zS2.sfx +MY_FIXED = 1 C_OBJS = \ $O\7zAlloc.obj \ @@ -16,6 +17,7 @@ C_OBJS = \ $O\BraIA64.obj \ $O\CpuArch.obj \ $O\Delta.obj \ + $O\DllSecur.obj \ $O\Lzma2Dec.obj \ $O\LzmaDec.obj \ diff --git a/src/third-party/lzma/Util/SfxSetup/makefile_con b/src/third-party/lzma/Util/SfxSetup/makefile_con index 173bdbf1d..6f604ed81 100644 --- a/src/third-party/lzma/Util/SfxSetup/makefile_con +++ b/src/third-party/lzma/Util/SfxSetup/makefile_con @@ -1,4 +1,5 @@ PROG = 7zS2con.sfx +MY_FIXED = 1 CFLAGS = $(CFLAGS) -D_CONSOLE C_OBJS = \ @@ -17,6 +18,7 @@ C_OBJS = \ $O\BraIA64.obj \ $O\CpuArch.obj \ $O\Delta.obj \ + $O\DllSecur.obj \ $O\Lzma2Dec.obj \ $O\LzmaDec.obj \ diff --git a/src/third-party/lzma/Xz.c b/src/third-party/lzma/Xz.c index 92144db18..7e061d6e7 100644 --- a/src/third-party/lzma/Xz.c +++ b/src/third-party/lzma/Xz.c @@ -1,5 +1,5 @@ /* Xz.c - Xz -2015-05-01 : Igor Pavlov : Public domain */ +2017-05-12 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,7 +9,7 @@ #include "XzCrc64.h" const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; -const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; +/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) { @@ -20,22 +20,22 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) v >>= 7; } while (v != 0); - buf[i - 1] &= 0x7F; + buf[(size_t)i - 1] &= 0x7F; return i; } void Xz_Construct(CXzStream *p) { - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + p->numBlocks = 0; + p->blocks = NULL; p->flags = 0; } -void Xz_Free(CXzStream *p, ISzAlloc *alloc) +void Xz_Free(CXzStream *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->blocks); - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + ISzAlloc_Free(alloc, p->blocks); + p->numBlocks = 0; + p->blocks = NULL; } unsigned XzFlags_GetCheckSize(CXzStreamFlags f) diff --git a/src/third-party/lzma/Xz.h b/src/third-party/lzma/Xz.h index 6937beaf6..fad56a3fb 100644 --- a/src/third-party/lzma/Xz.h +++ b/src/third-party/lzma/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2015-05-01 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -50,9 +50,10 @@ typedef struct #define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) #define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) +#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */ @@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt #define XZ_FOOTER_SIG_SIZE 2 extern const Byte XZ_SIG[XZ_SIG_SIZE]; + +/* extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; +*/ + +#define XZ_FOOTER_SIG_0 'Y' +#define XZ_FOOTER_SIG_1 'Z' #define XZ_STREAM_FLAGS_SIZE 2 #define XZ_STREAM_CRC_SIZE 4 @@ -106,13 +113,12 @@ typedef struct { CXzStreamFlags flags; size_t numBlocks; - size_t numBlocksAllocated; CXzBlockSizes *blocks; UInt64 startOffset; } CXzStream; void Xz_Construct(CXzStream *p); -void Xz_Free(CXzStream *p, ISzAlloc *alloc); +void Xz_Free(CXzStream *p, ISzAllocPtr alloc); #define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1) @@ -127,12 +133,15 @@ typedef struct } CXzs; void Xzs_Construct(CXzs *p); -void Xzs_Free(CXzs *p, ISzAlloc *alloc); -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc); +void Xzs_Free(CXzs *p, ISzAllocPtr alloc); +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ @@ -141,43 +150,55 @@ typedef enum CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; - void (*Free)(void *p, ISzAlloc *alloc); - SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc); + void (*Free)(void *p, ISzAllocPtr alloc); + SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct { - ISzAlloc *alloc; + ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + BoolInt wasFinished; + SRes res; + ECoderStatus status; + // BoolInt SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum { @@ -191,6 +212,7 @@ typedef enum XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state; @@ -204,7 +226,7 @@ typedef struct UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize; @@ -218,14 +240,65 @@ typedef struct CXzBlock block; CXzCheck check; CSha256 sha; + + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; + unsigned decodeOnlyOneBlock; + + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc); +/* alloc : aligned for cache line allocation is better */ +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); +/* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + /* finishMode: It has meaning only if the decoding reaches output limit (*destLen). @@ -255,20 +328,132 @@ Returns: SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p); +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p); +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); + + +/* + for random block decoding: + XzUnpacker_Init(); + set CXzUnpacker::streamFlags + XzUnpacker_PrepareToRandomBlockDecoding() + loop + { + XzUnpacker_Code() + XzUnpacker_IsBlockFinished() + } +*/ + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); + +#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + BoolInt ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); EXTERN_C_END diff --git a/src/third-party/lzma/XzCrc64.c b/src/third-party/lzma/XzCrc64.c index bd0cf5a37..e9ca9ec26 100644 --- a/src/third-party/lzma/XzCrc64.c +++ b/src/third-party/lzma/XzCrc64.c @@ -1,5 +1,5 @@ /* XzCrc64.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-05-23 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,9 +9,9 @@ #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) #ifdef MY_CPU_LE - #define CRC_NUM_TABLES 4 + #define CRC64_NUM_TABLES 4 #else - #define CRC_NUM_TABLES 5 + #define CRC64_NUM_TABLES 5 #define CRC_UINT64_SWAP(v) \ ((v >> 56) \ | ((v >> 40) & ((UInt64)0xFF << 8)) \ @@ -29,10 +29,10 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); -static CRC_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC_NUM_TABLES]; +static CRC64_FUNC g_Crc64Update; +UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) { @@ -52,12 +52,12 @@ void MY_FAST_CALL Crc64GenerateTable() UInt64 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrc64Poly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) { - UInt64 r = g_Crc64Table[i - 256]; + UInt64 r = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); } @@ -74,9 +74,9 @@ void MY_FAST_CALL Crc64GenerateTable() else #endif { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) { - UInt64 x = g_Crc64Table[i - 256]; + UInt64 x = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = CRC_UINT64_SWAP(x); } g_Crc64Update = XzCrc64UpdateT1_BeT4; diff --git a/src/third-party/lzma/XzCrc64Opt.c b/src/third-party/lzma/XzCrc64Opt.c index 97d086e6a..9273465d4 100644 --- a/src/third-party/lzma/XzCrc64Opt.c +++ b/src/third-party/lzma/XzCrc64Opt.c @@ -1,5 +1,5 @@ /* XzCrc64Opt.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-06-30 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,24 +7,24 @@ #ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)v ^ *(const UInt32 *)p; v = (v >> 32) - ^ table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + ^ (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); return v; } @@ -43,7 +43,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con | ((v << 40) & ((UInt64)0xFF << 48)) \ | ((v << 56))) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { @@ -51,18 +51,18 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size table += 0x100; v = CRC_UINT64_SWAP(v); for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; v = (v << 32) - ^ table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + ^ (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); return CRC_UINT64_SWAP(v); } diff --git a/src/third-party/lzma/XzDec.c b/src/third-party/lzma/XzDec.c index 29831b0ca..4f5327207 100644 --- a/src/third-party/lzma/XzDec.c +++ b/src/third-party/lzma/XzDec.c @@ -1,14 +1,33 @@ /* XzDec.c -- Xz Decode -2015-11-09 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include #include @@ -19,16 +38,18 @@ #include "Delta.h" #include "Lzma2Dec.h" +// #define USE_SUBBLOCK + #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" #define XZ_CHECK_SIZE_MAX 64 -#define CODER_BUF_SIZE (1 << 17) +#define CODER_BUF_SIZE ((size_t)1 << 17) unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) { @@ -56,8 +77,9 @@ typedef struct size_t bufConv; size_t bufTotal; - UInt32 methodId; int encodeMode; + + UInt32 methodId; UInt32 delta; UInt32 ip; UInt32 x86State; @@ -66,12 +88,12 @@ typedef struct Byte buf[BRA_BUF_SIZE]; } CBraState; -static void BraState_Free(void *pp, ISzAlloc *alloc) +static void BraState_Free(void *pp, ISzAllocPtr alloc) { - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { CBraState *p = ((CBraState *)pp); UNUSED_VAR(alloc); @@ -121,69 +143,84 @@ static void BraState_Init(void *pp) Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -192,48 +229,54 @@ static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)alloc->Alloc(alloc, sizeof(CBraState)); - if (decoder == 0) - return SZ_ERROR_MEM; + decoder = (CBraState *)p->p; + if (!decoder) + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK -static void SbState_Free(void *pp, ISzAlloc *alloc) +static void SbState_Free(void *pp, ISzAllocPtr alloc) { CSbDec *p = (CSbDec *)pp; SbDec_Free(p); - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { UNUSED_VAR(pp); UNUSED_VAR(props); @@ -246,8 +289,10 @@ static void SbState_Init(void *pp) SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res; @@ -260,102 +305,185 @@ static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = alloc->Alloc(alloc, sizeof(CSbDec)); - if (decoder == 0) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + CSbDec *decoder = (CSbDec *)p->p; + if (!decoder) + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ -static void Lzma2State_Free(void *pp, ISzAlloc *alloc) + +/* ---------- Lzma2 ---------- */ + +typedef struct { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); - alloc->Free(alloc, pp); + CLzma2Dec decoder; + BoolInt outBufMode; +} CLzma2Dec_Spec; + + +static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) +{ + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); + ISzAlloc_Free(alloc, pp); } -static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = (ECoderStatus)status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) { - CLzma2Dec *decoder = (CLzma2Dec *)alloc->Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (decoder == 0) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc) +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { - p->alloc->Free(p->alloc, p->buf); + ISzAlloc_Free(p->alloc, p->buf); p->buf = NULL; /* 9.31: the BUG was fixed */ } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) @@ -368,16 +496,22 @@ void MixCoder_Init(CMixCoder *p) { IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif @@ -387,31 +521,128 @@ SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; - Bool allFinished = True; + *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)p->alloc->Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { - Bool processed = False; + BoolInt processed = False; + BoolInt allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break; @@ -421,79 +652,106 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { - srcCur = p->buf + (CODER_BUF_SIZE * (i - 1)) + p->pos[i - 1]; - srcLenCur = p->size[i - 1] - p->pos[i - 1]; - srcFinishedCur = p->finished[i - 1]; + size_t k = i - 1; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } + if (!processed) - break; + { + if (allFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; + } } - if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; } + SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) { *p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE); @@ -503,13 +761,13 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } -static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { - return - indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && - (GetUi32(buf) == CrcCalc(buf + 4, 6) && - flags == GetBe16(buf + 8) && - memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0); + return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) + && GetUi32(buf) == CrcCalc(buf + 4, 6) + && flags == GetBe16(buf + 8) + && buf[10] == XZ_FOOTER_SIG_0 + && buf[11] == XZ_FOOTER_SIG_1; } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ @@ -517,20 +775,48 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + SRes XzBlock_Parse(CXzBlock *p, const Byte *header) { unsigned pos; unsigned numFilters, i; unsigned headerSize = (unsigned)header[0] << 2; + /* (headerSize != 0) : another code checks */ + if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) return SZ_ERROR_ARCHIVE; pos = 1; - if (pos == headerSize) - return SZ_ERROR_ARCHIVE; p->flags = header[pos++]; + p->packSize = (UInt64)(Int64)-1; if (XzBlock_HasPackSize(p)) { READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); @@ -538,6 +824,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_ERROR_ARCHIVE; } + p->unpackSize = (UInt64)(Int64)-1; if (XzBlock_HasUnpackSize(p)) READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); @@ -564,37 +851,56 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) #endif } + if (XzBlock_HasUnsupportedFlags(p)) + return SZ_ERROR_UNSUPPORTED; + while (pos < headerSize) if (header[pos++] != 0) return SZ_ERROR_ARCHIVE; return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; - Bool needReInit = True; + BoolInt needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; @@ -606,6 +912,8 @@ SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER; @@ -614,75 +922,171 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numFinishedStreams = 0; p->numTotalBlocks = 0; p->padSize = 0; + p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; } -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc) + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; +} + + +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); } + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) +{ + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + p->decodeOnlyOneBlock = 1; +} + + +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { - SizeT srcRem = srcLenOrig - *srcLen; + SizeT srcRem; if (p->state == XZ_STATE_BLOCK) { SizeT destLen2 = destLenOrig - *destLen; SizeT srcLen2 = srcLenOrig - *srcLen; SRes res; + + ECoderFinishMode finishMode2 = finishMode; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; + + if (p->block.packSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.packSize - p->packSize; + if (srcLen2 >= rem) + { + srcFinished2 = True; + srcLen2 = (SizeT)rem; + } + if (rem == 0 && p->block.unpackSize == p->unpackSize) + return SZ_ERROR_DATA; + } + + if (p->block.unpackSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.unpackSize - p->unpackSize; + if (destLen2 >= rem) + { + destFinish = True; + finishMode2 = CODER_FINISH_END; + destLen2 = (SizeT)rem; + } + } + + /* if (srcLen2 == 0 && destLen2 == 0) { *status = CODER_STATUS_NOT_FINISHED; return SZ_OK; } + */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; - + RINOK(res); - - if (*status == CODER_STATUS_FINISHED_WITH_MARK) + + if (*status != CODER_STATUS_FINISHED_WITH_MARK) { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + if (p->block.packSize == p->packSize + && *status == CODER_STATUS_NEEDS_MORE_INPUT) + { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); + *status = CODER_STATUS_NOT_SPECIFIED; + return SZ_ERROR_DATA; + } + return SZ_OK; + } + { + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } - else if (srcLen2 == 0 && destLen2 == 0) - return SZ_OK; - - continue; + // continue; } - if (srcRem == 0) + srcRem = srcLenOrig - *srcLen; + + // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes + if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) { *status = CODER_STATUS_NEEDS_MORE_INPUT; return SZ_OK; @@ -696,6 +1100,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; } @@ -703,10 +1109,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); p->numStartedStreams++; - p->state = XZ_STATE_BLOCK_HEADER; - Sha256_Init(&p->sha); p->indexSize = 0; p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; } break; @@ -720,6 +1126,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (*srcLen)++; if (p->buf[0] == 0) { + if (p->decodeOnlyOneBlock) + return SZ_ERROR_DATA; p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); p->indexPos = p->indexPreSize; p->indexSize += p->indexPreSize; @@ -727,10 +1135,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, Sha256_Init(&p->sha); p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); p->state = XZ_STATE_STREAM_INDEX; + break; } p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + break; } - else if (p->pos != p->blockHeaderSize) + + if (p->pos != p->blockHeaderSize) { UInt32 cur = p->blockHeaderSize - p->pos; if (cur > srcRem) @@ -743,20 +1154,32 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } case XZ_STATE_BLOCK_FOOTER: { - if (((p->packSize + p->alignPos) & 3) != 0) + if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } (*srcLen)++; p->alignPos++; if (*src++ != 0) @@ -768,20 +1191,31 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt32 cur = checkSize - p->pos; if (cur != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } if (cur > srcRem) cur = (UInt32)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; src += cur; + if (checkSize != p->pos) + break; } - else { Byte digest[XZ_CHECK_SIZE_MAX]; p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) return SZ_ERROR_CRC; + if (p->decodeOnlyOneBlock) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } } } break; @@ -897,17 +1331,1436 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, */ } -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p) + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); +} + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); } -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p) +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + BoolInt skipMode; + // BoolInt finishedWithMark; + EMtDecParseState parseState; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + BoolInt dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + BoolInt readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + BoolInt mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + BoolInt parsing_Truncated; + + BoolInt mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + XzUnpacker_Free(&p->dec); + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + BoolInt finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + BoolInt decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + XzUnpacker_Init(&p->dec); + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + BoolInt needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + BoolInt truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} diff --git a/src/third-party/lzma/XzEnc.c b/src/third-party/lzma/XzEnc.c index 2e5a003ad..309eca949 100644 --- a/src/third-party/lzma/XzEnc.c +++ b/src/third-party/lzma/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2015-09-16 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,7 +7,6 @@ #include #include "7zCrc.h" -#include "Alloc.h" #include "Bra.h" #include "CpuArch.h" @@ -19,22 +18,41 @@ #include "XzEnc.h" +// #define _7ZIP_ST + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#define MTCODER__BLOCKS_MAX 1 +#endif + +#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) + +/* max pack size for LZMA2 block + check-64bytrs: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) + +#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) + + #define XzBlock_ClearFlags(p) (p)->flags = 0; #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; -static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) + +static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) { - return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; + return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; } -static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) +static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) { *crc = CrcUpdate(*crc, buf, size); return WriteBytes(s, buf, size); } + static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) { UInt32 crc; @@ -78,98 +96,154 @@ static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) } -static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) + + +typedef struct +{ + size_t numBlocks; + size_t size; + size_t allocated; + Byte *blocks; +} CXzEncIndex; + + +static void XzEncIndex_Construct(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; + p->blocks = NULL; +} + +static void XzEncIndex_Init(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; +} + +static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) +{ + if (p->blocks) + { + ISzAlloc_Free(alloc, p->blocks); + p->blocks = NULL; + } + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; +} + + +static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) +{ + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!blocks) + return SZ_ERROR_MEM; + if (p->size != 0) + memcpy(blocks, p->blocks, p->size); + if (p->blocks) + ISzAlloc_Free(alloc, p->blocks); + p->blocks = blocks; + p->allocated = newSize; + return SZ_OK; +} + + +static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + UInt64 pos; + { + Byte buf[32]; + unsigned pos2 = Xz_WriteVarInt(buf, totalSize); + pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); + pos = numBlocks * pos2; + } + + if (pos <= p->allocated - p->size) + return SZ_OK; + { + UInt64 newSize64 = p->size + pos; + size_t newSize = (size_t)newSize64; + if (newSize != newSize64) + return SZ_ERROR_MEM; + return XzEncIndex_ReAlloc(p, newSize, alloc); + } +} + + +static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + Byte buf[32]; + unsigned pos = Xz_WriteVarInt(buf, totalSize); + pos += Xz_WriteVarInt(buf + pos, unpackSize); + + if (pos > p->allocated - p->size) + { + size_t newSize = p->allocated * 2 + 16 * 2; + if (newSize < p->size + pos) + return SZ_ERROR_MEM; + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); + } + memcpy(p->blocks + p->size, buf, pos); + p->size += pos; + p->numBlocks++; + return SZ_OK; +} + + +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) { Byte buf[32]; UInt64 globalPos; - { - UInt32 crc = CRC_INIT_VAL; - unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); - size_t i; - - globalPos = pos; - buf[0] = 0; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - - for (i = 0; i < p->numBlocks; i++) - { - const CXzBlockSizes *block = &p->blocks[i]; - pos = Xz_WriteVarInt(buf, block->totalSize); - pos += Xz_WriteVarInt(buf + pos, block->unpackSize); - globalPos += pos; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - } - - pos = ((unsigned)globalPos & 3); - - if (pos != 0) - { - buf[0] = buf[1] = buf[2] = 0; - RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); - globalPos += 4 - pos; - } - { - SetUi32(buf, CRC_GET_DIGEST(crc)); - RINOK(WriteBytes(s, buf, 4)); - globalPos += 4; - } - } - - { - UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); - SetUi32(buf + 4, indexSize); - buf[8] = (Byte)(p->flags >> 8); - buf[9] = (Byte)(p->flags & 0xFF); - SetUi32(buf, CrcCalc(buf + 4, 6)); - memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); - return WriteBytes(s, buf, 12); - } + UInt32 crc = CRC_INIT_VAL; + unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); + + globalPos = pos; + buf[0] = 0; + RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); + RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + globalPos += p->size; + + pos = XZ_GET_PAD_SIZE(globalPos); + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + globalPos += pos; + + crc = CrcUpdate(crc, buf + 4 - pos, pos); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + buf[8 + 8] = (Byte)(flags >> 8); + buf[8 + 9] = (Byte)(flags & 0xFF); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + buf[8 + 10] = XZ_FOOTER_SIG_0; + buf[8 + 11] = XZ_FOOTER_SIG_1; + + return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); } -static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc) -{ - if (!p->blocks || p->numBlocksAllocated == p->numBlocks) - { - size_t num = p->numBlocks * 2 + 1; - size_t newSize = sizeof(CXzBlockSizes) * num; - CXzBlockSizes *blocks; - if (newSize / sizeof(CXzBlockSizes) != num) - return SZ_ERROR_MEM; - blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize); - if (!blocks) - return SZ_ERROR_MEM; - if (p->numBlocks != 0) - { - memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); - alloc->Free(alloc, p->blocks); - } - p->blocks = blocks; - p->numBlocksAllocated = num; - } - { - CXzBlockSizes *block = &p->blocks[p->numBlocks++]; - block->unpackSize = unpackSize; - block->totalSize = totalSize; - } - return SZ_OK; -} - /* ---------- CSeqCheckInStream ---------- */ typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *realStream; + const Byte *data; + UInt64 limit; UInt64 processed; + int realStreamFinished; CXzCheck check; } CSeqCheckInStream; -static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode) +static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) { + p->limit = (UInt64)(Int64)-1; p->processed = 0; - XzCheck_Init(&p->check, mode); + p->realStreamFinished = 0; + XzCheck_Init(&p->check, checkMode); } static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) @@ -177,12 +251,31 @@ static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) XzCheck_Final(&p->check, digest); } -static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) +static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSeqCheckInStream *p = (CSeqCheckInStream *)pp; - SRes res = p->realStream->Read(p->realStream, data, size); - XzCheck_Update(&p->check, data, *size); - p->processed += *size; + CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + if (p->realStream) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->realStreamFinished = (size2 == 0) ? 1 : 0; + } + else + memcpy(data, p->data + (size_t)p->processed, size2); + XzCheck_Update(&p->check, data, size2); + p->processed += size2; + } + *size = size2; return res; } @@ -191,15 +284,24 @@ static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) typedef struct { - ISeqOutStream p; + ISeqOutStream vt; ISeqOutStream *realStream; + Byte *outBuf; + size_t outBufLimit; UInt64 processed; } CSeqSizeOutStream; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp; - size = p->realStream->Write(p->realStream, data, size); + CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); + if (p->realStream) + size = ISeqOutStream_Write(p->realStream, data, size); + else + { + if (size > p->outBufLimit - (size_t)p->processed) + return 0; + memcpy(p->outBuf + (size_t)p->processed, data, size); + } p->processed += size; return size; } @@ -220,9 +322,29 @@ typedef struct int srcWasFinished; } CSeqInFilter; -static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); + +static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) { - CSeqInFilter *p = (CSeqInFilter *)pp; + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + if (!p->buf) + return SZ_ERROR_MEM; + } + p->curPos = p->endPos = 0; + p->srcWasFinished = 0; + RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + p->StateCoder.Init(p->StateCoder.p); + return SZ_OK; +} + + +static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); size_t sizeOriginal = *size; if (sizeOriginal == 0) return SZ_OK; @@ -234,19 +356,22 @@ static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) { p->curPos = 0; p->endPos = FILTER_BUF_SIZE; - RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos)); + RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); if (p->endPos == 0) p->srcWasFinished = 1; } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + (Byte *)data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } } @@ -255,36 +380,24 @@ static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) static void SeqInFilter_Construct(CSeqInFilter *p) { p->buf = NULL; + p->StateCoder.p = NULL; p->p.Read = SeqInFilter_Read; } -static void SeqInFilter_Free(CSeqInFilter *p) +static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) { + if (p->StateCoder.p) + { + p->StateCoder.Free(p->StateCoder.p, alloc); + p->StateCoder.p = NULL; + } if (p->buf) { - g_Alloc.Free(&g_Alloc, p->buf); + ISzAlloc_Free(alloc, p->buf); p->buf = NULL; } } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc); - -static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) -{ - if (!p->buf) - { - p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE); - if (!p->buf) - return SZ_ERROR_MEM; - } - p->curPos = p->endPos = 0; - p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc)); - p->StateCoder.Init(p->StateCoder.p); - return SZ_OK; -} - /* ---------- CSbEncInStream ---------- */ @@ -292,17 +405,17 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *inStream; CSbEnc enc; } CSbEncInStream; -static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) +static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSbEncInStream *p = (CSbEncInStream *)pp; + CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); size_t sizeOriginal = *size; if (sizeOriginal == 0) - return S_OK; + return SZ_OK; for (;;) { @@ -322,14 +435,14 @@ static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) *size = sizeOriginal; RINOK(SbEnc_Read(&p->enc, data, size)); if (*size != 0 || !p->enc.needRead) - return S_OK; + return SZ_OK; } } -void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc) +void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) { SbEnc_Construct(&p->enc, alloc); - p->p.Read = SbEncInStream_Read; + p->vt.Read = SbEncInStream_Read; } SRes SbEncInStream_Init(CSbEncInStream *p) @@ -345,57 +458,9 @@ void SbEncInStream_Free(CSbEncInStream *p) #endif -typedef struct -{ - CLzma2EncHandle lzma2; - #ifdef USE_SUBBLOCK - CSbEncInStream sb; - #endif - CSeqInFilter filter; - ISzAlloc *alloc; - ISzAlloc *bigAlloc; -} CLzma2WithFilters; +/* ---------- CXzProps ---------- */ -static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc) -{ - p->alloc = alloc; - p->bigAlloc = bigAlloc; - p->lzma2 = NULL; - #ifdef USE_SUBBLOCK - SbEncInStream_Construct(&p->sb, alloc); - #endif - SeqInFilter_Construct(&p->filter); -} - -static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) -{ - p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); - if (!p->lzma2) - return SZ_ERROR_MEM; - return SZ_OK; -} - -static void Lzma2WithFilters_Free(CLzma2WithFilters *p) -{ - SeqInFilter_Free(&p->filter); - #ifdef USE_SUBBLOCK - SbEncInStream_Free(&p->sb); - #endif - if (p->lzma2) - { - Lzma2Enc_Destroy(p->lzma2); - p->lzma2 = NULL; - } -} - - -void XzProps_Init(CXzProps *p) -{ - p->lzma2Props = NULL; - p->filterProps = NULL; - p->checkId = XZ_CHECK_CRC32; -} void XzFilterProps_Init(CXzFilterProps *p) { @@ -405,122 +470,848 @@ void XzFilterProps_Init(CXzFilterProps *p) p->ipDefined = False; } - -static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf, - ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) +void XzProps_Init(CXzProps *p) { - xz->flags = (Byte)props->checkId; + p->checkId = XZ_CHECK_CRC32; + p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; + p->reduceSize = (UInt64)(Int64)-1; + p->forceWriteSizesInHeader = 0; + // p->forceWriteSizesInHeader = 1; - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props)); - RINOK(Xz_WriteHeader(xz->flags, outStream)); - - { - CSeqCheckInStream checkInStream; - CSeqSizeOutStream seqSizeOutStream; - CXzBlock block; - unsigned filterIndex = 0; - CXzFilter *filter = NULL; - const CXzFilterProps *fp = props->filterProps; - - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); - - if (fp) - { - filter = &block.filters[filterIndex++]; - filter->id = fp->id; - filter->propsSize = 0; - - if (fp->id == XZ_ID_Delta) - { - filter->props[0] = (Byte)(fp->delta - 1); - filter->propsSize = 1; - } - else if (fp->ipDefined) - { - SetUi32(filter->props, fp->ip); - filter->propsSize = 4; - } - } - - { - CXzFilter *f = &block.filters[filterIndex++]; - f->id = XZ_ID_LZMA2; - f->propsSize = 1; - f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); - } - - seqSizeOutStream.p.Write = MyWrite; - seqSizeOutStream.realStream = outStream; - seqSizeOutStream.processed = 0; - - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p)); - - checkInStream.p.Read = SeqCheckInStream_Read; - checkInStream.realStream = inStream; - SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); - - if (fp) - { - #ifdef USE_SUBBLOCK - if (fp->id == XZ_ID_Subblock) - { - lzmaf->sb.inStream = &checkInStream.p; - RINOK(SbEncInStream_Init(&lzmaf->sb)); - } - else - #endif - { - lzmaf->filter.realStream = &checkInStream.p; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter)); - } - } - - { - UInt64 packPos = seqSizeOutStream.processed; - - SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p, - fp ? - #ifdef USE_SUBBLOCK - (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p: - #endif - &lzmaf->filter.p: - &checkInStream.p, - progress); - - RINOK(res); - block.unpackSize = checkInStream.processed; - block.packSize = seqSizeOutStream.processed - packPos; - } - - { - unsigned padSize = 0; - Byte buf[128]; - while ((((unsigned)block.packSize + padSize) & 3) != 0) - buf[padSize++] = 0; - SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); - RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags))); - RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); - } - } - return Xz_WriteFooter(xz, outStream); + XzFilterProps_Init(&p->filterProps); + Lzma2EncProps_Init(&p->lzma2Props); } +static void XzEncProps_Normalize_Fixed(CXzProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + Lzma2EncProps_Normalize(&tp); + t1n = tp.numTotalThreads; + } + + t1 = p->lzma2Props.numTotalThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzma2Props.numTotalThreads = t1; + + t2r = t2; + + fileSize = p->reduceSize; + + if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzma2Props.lzmaProps.reduceSize = p->blockSize; + + Lzma2EncProps_Normalize(&p->lzma2Props); + + t1 = p->lzma2Props.numTotalThreads; + + { + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static void XzProps_Normalize(CXzProps *p) +{ + /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. + Lzma2Enc_SetProps() will normalize lzma2Props later. */ + + if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + { + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + return; + } + else + { + CLzma2EncProps *lzma2 = &p->lzma2Props; + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + // xz-auto + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + // if (xz-auto && lzma2-solid) - we use solid for both + p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + } + else + { + // if (xz-auto && (lzma2-auto || lzma2-fixed_) + // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + + Lzma2EncProps_Normalize(&tp); + + p->blockSize = tp.blockSize; // fixed or solid + p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; + p->numBlockThreads_Max = tp.numBlockThreads_Max; + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->lzmaProps.reduceSize = tp.blockSize; + lzma2->numBlockThreads_Reduced = 1; + lzma2->numBlockThreads_Max = 1; + return; + } + } + else + { + // xz-fixed + // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize + + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + { + UInt64 r = p->reduceSize; + if (r > p->blockSize || r == (UInt64)(Int64)-1) + r = p->blockSize; + lzma2->lzmaProps.reduceSize = r; + } + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->blockSize = p->blockSize; + + XzEncProps_Normalize_Fixed(p); + } + } +} + + +/* ---------- CLzma2WithFilters ---------- */ + +typedef struct +{ + CLzma2EncHandle lzma2; + CSeqInFilter filter; + + #ifdef USE_SUBBLOCK + CSbEncInStream sb; + #endif +} CLzma2WithFilters; + + +static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) +{ + p->lzma2 = NULL; + SeqInFilter_Construct(&p->filter); + + #ifdef USE_SUBBLOCK + SbEncInStream_Construct(&p->sb, alloc); + #endif +} + + +static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) +{ + if (!p->lzma2) + { + p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); + if (!p->lzma2) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + + +static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) +{ + #ifdef USE_SUBBLOCK + SbEncInStream_Free(&p->sb); + #endif + + SeqInFilter_Free(&p->filter, alloc); + if (p->lzma2) + { + Lzma2Enc_Destroy(p->lzma2); + p->lzma2 = NULL; + } +} + + +typedef struct +{ + UInt64 unpackSize; + UInt64 totalSize; + size_t headerSize; +} CXzEncBlockInfo; + + +static SRes Xz_CompressBlock( + CLzma2WithFilters *lzmaf, + + ISeqOutStream *outStream, + Byte *outBufHeader, + Byte *outBufData, size_t outBufDataLimit, + + ISeqInStream *inStream, + // UInt64 expectedSize, + const Byte *inBuf, // used if (!inStream) + size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored + + const CXzProps *props, + ICompressProgress *progress, + int *inStreamFinished, /* only for inStream version */ + CXzEncBlockInfo *blockSizes, + ISzAllocPtr alloc, + ISzAllocPtr allocBig) +{ + CSeqCheckInStream checkInStream; + CSeqSizeOutStream seqSizeOutStream; + CXzBlock block; + unsigned filterIndex = 0; + CXzFilter *filter = NULL; + const CXzFilterProps *fp = &props->filterProps; + if (fp->id == 0) + fp = NULL; + + *inStreamFinished = False; + + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + + XzBlock_ClearFlags(&block); + XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + + if (fp) + { + filter = &block.filters[filterIndex++]; + filter->id = fp->id; + filter->propsSize = 0; + + if (fp->id == XZ_ID_Delta) + { + filter->props[0] = (Byte)(fp->delta - 1); + filter->propsSize = 1; + } + else if (fp->ipDefined) + { + SetUi32(filter->props, fp->ip); + filter->propsSize = 4; + } + } + + { + CXzFilter *f = &block.filters[filterIndex++]; + f->id = XZ_ID_LZMA2; + f->propsSize = 1; + f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + } + + seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; + seqSizeOutStream.realStream = outStream; + seqSizeOutStream.outBuf = outBufData; + seqSizeOutStream.outBufLimit = outBufDataLimit; + seqSizeOutStream.processed = 0; + + /* + if (expectedSize != (UInt64)(Int64)-1) + { + block.unpackSize = expectedSize; + if (props->blockSize != (UInt64)(Int64)-1) + if (expectedSize > props->blockSize) + block.unpackSize = props->blockSize; + XzBlock_SetHasUnpackSize(&block); + } + */ + + if (outStream) + { + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + } + + checkInStream.vt.Read = SeqCheckInStream_Read; + SeqCheckInStream_Init(&checkInStream, props->checkId); + + checkInStream.realStream = inStream; + checkInStream.data = inBuf; + checkInStream.limit = props->blockSize; + if (!inStream) + checkInStream.limit = inBufSize; + + if (fp) + { + #ifdef USE_SUBBLOCK + if (fp->id == XZ_ID_Subblock) + { + lzmaf->sb.inStream = &checkInStream.vt; + RINOK(SbEncInStream_Init(&lzmaf->sb)); + } + else + #endif + { + lzmaf->filter.realStream = &checkInStream.vt; + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + } + } + + { + SRes res; + Byte *outBuf = NULL; + size_t outSize = 0; + BoolInt useStream = (fp || inStream); + // useStream = True; + + if (!useStream) + { + XzCheck_Update(&checkInStream.check, inBuf, inBufSize); + checkInStream.processed = inBufSize; + } + + if (!outStream) + { + outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; + outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; + } + + res = Lzma2Enc_Encode2(lzmaf->lzma2, + outBuf ? NULL : &seqSizeOutStream.vt, + outBuf, + outBuf ? &outSize : NULL, + + useStream ? + (fp ? + ( + #ifdef USE_SUBBLOCK + (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: + #endif + &lzmaf->filter.p) : + &checkInStream.vt) : NULL, + + useStream ? NULL : inBuf, + useStream ? 0 : inBufSize, + + progress); + + if (outBuf) + seqSizeOutStream.processed += outSize; + + RINOK(res); + blockSizes->unpackSize = checkInStream.processed; + } + { + Byte buf[4 + 64]; + unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + UInt64 packSize = seqSizeOutStream.processed; + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + SeqCheckInStream_GetDigest(&checkInStream, buf + 4); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + + blockSizes->totalSize = seqSizeOutStream.processed - padSize; + + if (!outStream) + { + seqSizeOutStream.outBuf = outBufHeader; + seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; + seqSizeOutStream.processed = 0; + + block.unpackSize = blockSizes->unpackSize; + XzBlock_SetHasUnpackSize(&block); + + block.packSize = packSize; + XzBlock_SetHasPackSize(&block); + + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + + blockSizes->headerSize = (size_t)seqSizeOutStream.processed; + blockSizes->totalSize += seqSizeOutStream.processed; + } + } + + if (inStream) + *inStreamFinished = checkInStream.realStreamFinished; + else + { + *inStreamFinished = False; + if (checkInStream.processed != inBufSize) + return SZ_ERROR_FAIL; + } + + return SZ_OK; +} + + + +typedef struct +{ + ICompressProgress vt; + ICompressProgress *progress; + UInt64 inOffset; + UInt64 outOffset; +} CCompressProgress_XzEncOffset; + + +static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + inSize += p->inOffset; + outSize += p->outOffset; + return ICompressProgress_Progress(p->progress, inSize, outSize); +} + + + + +typedef struct +{ + ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + CXzProps xzProps; + UInt64 expectedDataSize; + + CXzEncIndex xzIndex; + + CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + + size_t outBufSize; /* size of allocated outBufs[i] */ + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #ifndef _7ZIP_ST + unsigned checkType; + ISeqOutStream *outStream; + BoolInt mtCoder_WasConstructed; + CMtCoder mtCoder; + CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + #endif + +} CXzEnc; + + +static void XzEnc_Construct(CXzEnc *p) +{ + unsigned i; + + XzEncIndex_Construct(&p->xzIndex); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; + { + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif +} + + +static void XzEnc_FreeOutBufs(CXzEnc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + + +static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) +{ + unsigned i; + + XzEncIndex_Free(&p->xzIndex, alloc); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + XzEnc_FreeOutBufs(p); + #endif +} + + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); + if (!p) + return NULL; + XzEnc_Construct(p); + XzProps_Init(&p->xzProps); + XzProps_Normalize(&p->xzProps); + p->expectedDataSize = (UInt64)(Int64)-1; + p->alloc = alloc; + p->allocBig = allocBig; + return p; +} + + +void XzEnc_Destroy(CXzEncHandle pp) +{ + CXzEnc *p = (CXzEnc *)pp; + XzEnc_Free(p, p->alloc); + ISzAlloc_Free(p->alloc, p); +} + + +SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +{ + CXzEnc *p = (CXzEnc *)pp; + p->xzProps = *props; + XzProps_Normalize(&p->xzProps); + return SZ_OK; +} + + +void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +{ + CXzEnc *p = (CXzEnc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + + + +#ifndef _7ZIP_ST + +static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CXzEnc *me = (CXzEnc *)pp; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + UNUSED_VAR(finished) + + { + CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + bInfo->totalSize = 0; + bInfo->unpackSize = 0; + bInfo->headerSize = 0; + } + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + MtProgressThunk_Init(&progressThunk); + + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + res = Xz_CompressBlock( + &me->lzmaf_Items[coderIndex], + + NULL, + dest, + dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, + + NULL, + // srcSize, // expectedSize + src, srcSize, + + &me->xzProps, + &progressThunk.vt, + &inStreamFinished, + &blockSizes, + me->alloc, + me->allocBig); + + if (res == SZ_OK) + me->EncBlocks[outBufIndex] = blockSizes; + + return res; + } +} + + +static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CXzEnc *me = (CXzEnc *)pp; + + const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); + + { + UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + } + + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); +} + +#endif + + + +SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +{ + CXzEnc *p = (CXzEnc *)pp; + + const CXzProps *props = &p->xzProps; + + XzEncIndex_Init(&p->xzIndex); + { + UInt64 numBlocks = 1; + UInt64 blockSize = props->blockSize; + + if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + && props->reduceSize != (UInt64)(Int64)-1) + { + numBlocks = props->reduceSize / blockSize; + if (numBlocks * blockSize != props->reduceSize) + numBlocks++; + } + else + blockSize = (UInt64)1 << 62; + + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + } + + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + + + #ifndef _7ZIP_ST + if (props->numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = XzEnc_MtCallback_Code; + vt.Write = XzEnc_MtCallback_Write; + + p->checkType = props->checkId; + p->xzProps = *props; + + p->outStream = outStream; + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = NULL; + p->mtCoder.inDataSize = 0; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID + || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + return SZ_ERROR_FAIL; + + p->mtCoder.blockSize = (size_t)props->blockSize; + if (p->mtCoder.blockSize != props->blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + + { + size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + XzEnc_FreeOutBufs(p); + p->outBufSize = destBlockSize; + } + + p->mtCoder.numThreadsMax = props->numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; + + RINOK(MtCoder_Code(&p->mtCoder)); + } + else + #endif + { + int writeStartSizes; + CCompressProgress_XzEncOffset progress2; + Byte *bufData = NULL; + size_t bufSize = 0; + + progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; + progress2.inOffset = 0; + progress2.outOffset = 0; + progress2.progress = progress; + + writeStartSizes = 0; + + if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) + { + writeStartSizes = (props->forceWriteSizesInHeader > 0); + + if (writeStartSizes) + { + size_t t2; + size_t t = (size_t)props->blockSize; + if (t != props->blockSize) + return SZ_ERROR_PARAM; + t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); + if (t < props->blockSize) + return SZ_ERROR_PARAM; + t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; + if (!p->outBufs[0] || t2 != p->outBufSize) + { + XzEnc_FreeOutBufs(p); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); + if (!p->outBufs[0]) + return SZ_ERROR_MEM; + p->outBufSize = t2; + } + bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; + bufSize = t; + } + } + + for (;;) + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + /* + UInt64 rem = (UInt64)(Int64)-1; + if (props->reduceSize != (UInt64)(Int64)-1 + && props->reduceSize >= progress2.inOffset) + rem = props->reduceSize - progress2.inOffset; + */ + + blockSizes.headerSize = 0; // for GCC + + RINOK(Xz_CompressBlock( + &p->lzmaf_Items[0], + + writeStartSizes ? NULL : outStream, + writeStartSizes ? p->outBufs[0] : NULL, + bufData, bufSize, + + inStream, + // rem, + NULL, 0, + + props, + progress ? &progress2.vt : NULL, + &inStreamFinished, + &blockSizes, + p->alloc, + p->allocBig)); + + { + UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); + + if (writeStartSizes) + { + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + } + + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + + progress2.inOffset += blockSizes.unpackSize; + progress2.outOffset += totalPackFull; + } + + if (inStreamFinished) + break; + } + } + + return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); +} + + +#include "Alloc.h" + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress) { SRes res; - CXzStream xz; - CLzma2WithFilters lzmaf; - Xz_Construct(&xz); - Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); - res = Lzma2WithFilters_Create(&lzmaf); + CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!xz) + return SZ_ERROR_MEM; + res = XzEnc_SetProps(xz, props); if (res == SZ_OK) - res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress); - Lzma2WithFilters_Free(&lzmaf); - Xz_Free(&xz, &g_Alloc); + res = XzEnc_Encode(xz, outStream, inStream, progress); + XzEnc_Destroy(xz); return res; } @@ -528,11 +1319,11 @@ SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, SRes Xz_EncodeEmpty(ISeqOutStream *outStream) { SRes res; - CXzStream xz; - Xz_Construct(&xz); - res = Xz_WriteHeader(xz.flags, outStream); + CXzEncIndex xzIndex; + XzEncIndex_Construct(&xzIndex); + res = Xz_WriteHeader((CXzStreamFlags)0, outStream); if (res == SZ_OK) - res = Xz_WriteFooter(&xz, outStream); - Xz_Free(&xz, &g_Alloc); + res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); + XzEncIndex_Free(&xzIndex, NULL); // g_Alloc return res; } diff --git a/src/third-party/lzma/XzEnc.h b/src/third-party/lzma/XzEnc.h index e9cea34f7..529ac3fd8 100644 --- a/src/third-party/lzma/XzEnc.h +++ b/src/third-party/lzma/XzEnc.h @@ -1,5 +1,5 @@ /* XzEnc.h -- Xz Encode -2011-02-07 : Igor Pavlov : Public domain */ +2017-06-27 : Igor Pavlov : Public domain */ #ifndef __XZ_ENC_H #define __XZ_ENC_H @@ -10,6 +10,11 @@ EXTERN_C_BEGIN + +#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO +#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + + typedef struct { UInt32 id; @@ -20,15 +25,31 @@ typedef struct void XzFilterProps_Init(CXzFilterProps *p); + typedef struct { - const CLzma2EncProps *lzma2Props; - const CXzFilterProps *filterProps; + CLzma2EncProps lzma2Props; + CXzFilterProps filterProps; unsigned checkId; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; + int forceWriteSizesInHeader; + UInt64 reduceSize; } CXzProps; void XzProps_Init(CXzProps *p); + +typedef void * CXzEncHandle; + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void XzEnc_Destroy(CXzEncHandle p); +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress); diff --git a/src/third-party/lzma/XzIn.c b/src/third-party/lzma/XzIn.c index aaa059730..792a61786 100644 --- a/src/third-party/lzma/XzIn.c +++ b/src/third-party/lzma/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,12 @@ #include "CpuArch.h" #include "Xz.h" +/* +#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) +*/ +#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) + + SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { Byte sig[XZ_STREAM_HEADER_SIZE]; @@ -22,13 +28,13 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; *headerSizeRes = 0; RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = ((unsigned)header[0] << 2) + 4; + headerSize = (unsigned)header[0]; if (headerSize == 0) { *headerSizeRes = 1; @@ -37,12 +43,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt } *isIndex = False; + headerSize = (headerSize << 2) + 4; *headerSizeRes = headerSize; RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); return XzBlock_Parse(p, header); } -#define ADD_SIZE_CHECH(size, val) \ +#define ADD_SIZE_CHECK(size, val) \ { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p) @@ -50,7 +57,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, p->blocks[i].unpackSize); + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); return size; } @@ -59,7 +66,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); return size; } @@ -70,7 +77,7 @@ SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStream *inStream) } */ -static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc *alloc) +static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { size_t numBlocks, pos = 1; UInt32 crc; @@ -96,9 +103,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc * { size_t i; p->numBlocks = numBlocks; - p->numBlocksAllocated = numBlocks; - p->blocks = alloc->Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); - if (p->blocks == 0) + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) { @@ -115,7 +121,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc * return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; } -static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAlloc *alloc) +static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) { SRes res; size_t size; @@ -125,13 +131,13 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = alloc->Alloc(alloc, size); - if (buf == 0) + buf = (Byte *)ISzAlloc_Alloc(alloc, size); + if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); if (res == SZ_OK) res = Xz_ReadIndex2(p, buf, size, alloc); - alloc->Free(alloc, buf); + ISzAlloc_Free(alloc, buf); return res; } @@ -142,7 +148,7 @@ static SRes LookInStream_SeekRead_ForArc(ILookInStream *stream, UInt64 offset, v /* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ } -static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAlloc *alloc) +static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) { UInt64 indexSize; Byte buf[XZ_STREAM_FOOTER_SIZE]; @@ -154,7 +160,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) { UInt32 total = 0; pos += XZ_STREAM_FOOTER_SIZE; @@ -187,7 +193,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) return SZ_ERROR_NO_ARCHIVE; } @@ -224,7 +230,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff SecToRead_CreateVTable(&secToRead); secToRead.realStream = stream; - RINOK(Xz_ReadHeader(&headerFlags, &secToRead.s)); + RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; } } @@ -238,12 +244,12 @@ void Xzs_Construct(CXzs *p) p->streams = 0; } -void Xzs_Free(CXzs *p, ISzAlloc *alloc) +void Xzs_Free(CXzs *p, ISzAllocPtr alloc) { size_t i; for (i = 0; i < p->num; i++) Xz_Free(&p->streams[i], alloc); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->num = p->numAllocated = 0; p->streams = 0; } @@ -262,7 +268,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); return size; } @@ -272,15 +278,15 @@ UInt64 Xzs_GetPackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); return size; } */ -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc) +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) { Int64 endOffset = 0; - RINOK(stream->Seek(stream, &endOffset, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); *startOffset = endOffset; for (;;) { @@ -293,20 +299,20 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr if (p->num == p->numAllocated) { size_t newNum = p->num + p->num / 4 + 1; - Byte *data = (Byte *)alloc->Alloc(alloc, newNum * sizeof(CXzStream)); - if (data == 0) + Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); + if (!data) return SZ_ERROR_MEM; p->numAllocated = newNum; if (p->num != 0) memcpy(data, p->streams, p->num * sizeof(CXzStream)); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->streams = (CXzStream *)data; } p->streams[p->num++] = st; if (*startOffset == 0) break; RINOK(LookInStream_SeekTo(stream, *startOffset)); - if (progress && progress->Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) + if (progress && ICompressProgress_Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) return SZ_ERROR_PROGRESS; } return SZ_OK; diff --git a/src/util/vfs/vfs-lzma.c b/src/util/vfs/vfs-lzma.c index e72f924a0..da2a6fe8e 100644 --- a/src/util/vfs/vfs-lzma.c +++ b/src/util/vfs/vfs-lzma.c @@ -16,6 +16,8 @@ #include "third-party/lzma/7zFile.h" #include "third-party/lzma/7zVersion.h" +#define BUFFER_SIZE 0x2000 + struct VDirEntry7z { struct VDirEntry d; @@ -30,7 +32,7 @@ struct VDir7z { // What is all this garbage? CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; ISzAlloc allocImp; ISzAlloc allocTempImp; @@ -88,15 +90,18 @@ struct VDir* VDirOpen7z(const char* path, int flags) { vd->allocTempImp.Free = SzFreeTemp; FileInStream_CreateVTable(&vd->archiveStream); - LookToRead_CreateVTable(&vd->lookStream, False); + LookToRead2_CreateVTable(&vd->lookStream, False); - vd->lookStream.realStream = &vd->archiveStream.s; - LookToRead_Init(&vd->lookStream); + vd->lookStream.realStream = &vd->archiveStream.vt; + vd->lookStream.buf = malloc(BUFFER_SIZE); + vd->lookStream.bufSize = BUFFER_SIZE; + + LookToRead2_Init(&vd->lookStream); CrcGenerateTable(); SzArEx_Init(&vd->db); - SRes res = SzArEx_Open(&vd->db, &vd->lookStream.s, &vd->allocImp, &vd->allocTempImp); + SRes res = SzArEx_Open(&vd->db, &vd->lookStream.vt, &vd->allocImp, &vd->allocTempImp); if (res != SZ_OK) { SzArEx_Free(&vd->db, &vd->allocImp); File_Close(&vd->archiveStream.file); @@ -211,6 +216,7 @@ bool _vd7zClose(struct VDir* vd) { SzArEx_Free(&vd7z->db, &vd7z->allocImp); File_Close(&vd7z->archiveStream.file); + free(vd7z->lookStream.buf); free(vd7z->dirent.utf8); vd7z->dirent.utf8 = 0; @@ -281,7 +287,7 @@ struct VFile* _vd7zOpenFile(struct VDir* vd, const char* path, int mode) { UInt32 blockIndex; vf->outBuffer = 0; - SRes res = SzArEx_Extract(&vd7z->db, &vd7z->lookStream.s, i, &blockIndex, + SRes res = SzArEx_Extract(&vd7z->db, &vd7z->lookStream.vt, i, &blockIndex, &vf->outBuffer, &outBufferSize, &vf->bufferOffset, &vf->size, &vd7z->allocImp, &vd7z->allocTempImp); From 4153a7e21f0bb6887ed8043a8a98d1803f628bf5 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 16 Jan 2020 20:10:54 -0800 Subject: [PATCH 15/20] Util: Fix memory leak in VDir7z --- src/util/vfs/vfs-lzma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/vfs/vfs-lzma.c b/src/util/vfs/vfs-lzma.c index da2a6fe8e..d56883c48 100644 --- a/src/util/vfs/vfs-lzma.c +++ b/src/util/vfs/vfs-lzma.c @@ -128,6 +128,7 @@ struct VDir* VDirOpen7z(const char* path, int flags) { bool _vf7zClose(struct VFile* vf) { struct VFile7z* vf7z = (struct VFile7z*) vf; IAlloc_Free(&vf7z->vd->allocImp, vf7z->outBuffer); + free(vf7z); return true; } From c0fa8256a2003941fdb9d23ca4ced1c8d2074a75 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Thu, 16 Jan 2020 20:51:17 -0800 Subject: [PATCH 16/20] All: Fix more warnings --- src/gb/audio.c | 2 +- src/gb/core.c | 2 +- src/gb/video.c | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/gb/audio.c b/src/gb/audio.c index 6bf7dde84..63af6bc1a 100644 --- a/src/gb/audio.c +++ b/src/gb/audio.c @@ -936,7 +936,7 @@ static void _updateChannel4(struct mTiming* timing, void* user, uint32_t cyclesL uint32_t last = 0; uint32_t now = cycles; - uint32_t next = cycles - cyclesLate; + int32_t next = cycles - cyclesLate; if (audio->style == GB_AUDIO_GBA) { last = ch->lastEvent; diff --git a/src/gb/core.c b/src/gb/core.c index 3647a043b..11d3f11ed 100644 --- a/src/gb/core.c +++ b/src/gb/core.c @@ -508,7 +508,7 @@ static void _GBCoreReset(struct mCore* core) { bios = NULL; } } - if (bios) { + if (found && bios) { GBLoadBIOS(gb, bios); } } diff --git a/src/gb/video.c b/src/gb/video.c index 4757a69d7..0859b13c9 100644 --- a/src/gb/video.c +++ b/src/gb/video.c @@ -187,7 +187,7 @@ void GBVideoAssociateRenderer(struct GBVideo* video, struct GBVideoRenderer* ren video->renderer->init(video->renderer, video->p->model, video->sgbBorders); } -static bool _statIRQAsserted(struct GBVideo* video, GBRegisterSTAT stat) { +static bool _statIRQAsserted(GBRegisterSTAT stat) { // TODO: variable for the IRQ line value? if (GBRegisterSTATIsLYCIRQ(stat) && GBRegisterSTATIsLYC(stat)) { return true; @@ -236,20 +236,20 @@ void _endMode0(struct mTiming* timing, void* context, uint32_t cyclesLate) { mTimingDeschedule(&video->p->timing, &video->frameEvent); mTimingSchedule(&video->p->timing, &video->frameEvent, -cyclesLate); - if (!_statIRQAsserted(video, oldStat) && GBRegisterSTATIsOAMIRQ(video->stat)) { + if (!_statIRQAsserted(oldStat) && GBRegisterSTATIsOAMIRQ(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); } video->p->memory.io[REG_IF] |= (1 << GB_IRQ_VBLANK); } video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); } // LYC stat is delayed 1 T-cycle oldStat = video->stat; video->stat = GBRegisterSTATSetLYC(video->stat, lyc == video->ly); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); } @@ -287,7 +287,7 @@ void _endMode1(struct mTiming* timing, void* context, uint32_t cyclesLate) { GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); video->stat = GBRegisterSTATSetLYC(video->stat, lyc == video->p->memory.io[REG_LY]); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); } @@ -305,7 +305,7 @@ void _endMode2(struct mTiming* timing, void* context, uint32_t cyclesLate) { video->modeEvent.callback = _endMode3; GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); } @@ -326,7 +326,7 @@ void _endMode3(struct mTiming* timing, void* context, uint32_t cyclesLate) { video->modeEvent.callback = _endMode0; GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); } @@ -416,7 +416,7 @@ void GBVideoWriteLCDC(struct GBVideo* video, GBRegisterLCDC value) { GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, 0); video->stat = GBRegisterSTATSetLYC(video->stat, video->ly == video->p->memory.io[REG_LYC]); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); } @@ -447,7 +447,7 @@ void GBVideoWriteSTAT(struct GBVideo* video, GBRegisterSTAT value) { if (!GBRegisterLCDCIsEnable(video->p->memory.io[REG_LCDC]) || video->p->model >= GB_MODEL_CGB) { return; } - if (!_statIRQAsserted(video, oldStat) && video->mode < 3) { + if (!_statIRQAsserted(oldStat) && video->mode < 3) { // TODO: variable for the IRQ line value? video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); @@ -458,7 +458,7 @@ void GBVideoWriteLYC(struct GBVideo* video, uint8_t value) { GBRegisterSTAT oldStat = video->stat; if (GBRegisterLCDCIsEnable(video->p->memory.io[REG_LCDC])) { video->stat = GBRegisterSTATSetLYC(video->stat, value == video->ly); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); } From 69a0372133bc1f7b806a23ecf89f8fb9c12f00c1 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Fri, 17 Jan 2020 20:44:00 -0800 Subject: [PATCH 17/20] GBA: Trim non-movie ROMs to 32 MiB if applicable --- CHANGES | 1 + src/gba/gba.c | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index ecf6a8af7..255c5aca2 100644 --- a/CHANGES +++ b/CHANGES @@ -118,6 +118,7 @@ Other fixes: - Wii: Improve audio buffering (fixes mgba.io/i/1617) Misc: - GB Memory: Support manual SRAM editing (fixes mgba.io/i/1580) + - GBA: Trim non-movie ROMs to 32 MiB if applicable - GBA Audio: Redo channel 4 batching for GBA only - GBA I/O: Stop logging several harmless invalid register reads - Debugger: Separate aliases from main commands diff --git a/src/gba/gba.c b/src/gba/gba.c index 84541dfa4..0513c7329 100644 --- a/src/gba/gba.c +++ b/src/gba/gba.c @@ -238,8 +238,16 @@ void GBAReset(struct ARMCore* cpu) { gba->debug = false; memset(gba->debugString, 0, sizeof(gba->debugString)); - if (gba->pristineRomSize > SIZE_CART0) { - GBAMatrixReset(gba); + + + if (gba->romVf && gba->pristineRomSize > SIZE_CART0) { + char ident; + gba->romVf->seek(gba->romVf, 0xAC, SEEK_SET); + gba->romVf->read(gba->romVf, &ident, 1); + gba->romVf->seek(gba->romVf, 0, SEEK_SET); + if (ident == 'M') { + GBAMatrixReset(gba); + } } } @@ -376,12 +384,21 @@ bool GBALoadROM(struct GBA* gba, struct VFile* vf) { vf->seek(vf, 0, SEEK_SET); if (gba->pristineRomSize > SIZE_CART0) { gba->isPristine = false; - gba->memory.romSize = 0x01000000; #ifdef FIXED_ROM_BUFFER gba->memory.rom = romBuffer; #else gba->memory.rom = anonymousMemoryMap(SIZE_CART0); #endif + char ident; + vf->seek(vf, 0xAC, SEEK_SET); + vf->read(vf, &ident, 1); + vf->seek(vf, 0, SEEK_SET); + if (ident == 'M') { + gba->memory.romSize = 0x01000000; + } else { + gba->memory.romSize = SIZE_CART0; + vf->read(vf, gba->memory.rom, SIZE_CART0); + } } else { gba->isPristine = true; gba->memory.rom = vf->map(vf, gba->pristineRomSize, MAP_READ); From 4ca16fb2ef9288d64f9c895888e28dd94e7e1a36 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 18 Jan 2020 19:19:42 -0800 Subject: [PATCH 18/20] GBA: Clean up previous commit --- src/gba/gba.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/gba/gba.c b/src/gba/gba.c index 0513c7329..7c00151ec 100644 --- a/src/gba/gba.c +++ b/src/gba/gba.c @@ -384,20 +384,19 @@ bool GBALoadROM(struct GBA* gba, struct VFile* vf) { vf->seek(vf, 0, SEEK_SET); if (gba->pristineRomSize > SIZE_CART0) { gba->isPristine = false; -#ifdef FIXED_ROM_BUFFER - gba->memory.rom = romBuffer; -#else - gba->memory.rom = anonymousMemoryMap(SIZE_CART0); -#endif char ident; vf->seek(vf, 0xAC, SEEK_SET); vf->read(vf, &ident, 1); - vf->seek(vf, 0, SEEK_SET); if (ident == 'M') { gba->memory.romSize = 0x01000000; +#ifdef FIXED_ROM_BUFFER + gba->memory.rom = romBuffer; +#else + gba->memory.rom = anonymousMemoryMap(SIZE_CART0); +#endif } else { + gba->memory.rom = vf->map(vf, SIZE_CART0, MAP_READ); gba->memory.romSize = SIZE_CART0; - vf->read(vf, gba->memory.rom, SIZE_CART0); } } else { gba->isPristine = true; From 95e1dd9c35c2fb4c5d942d65431aae4e3ece2142 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sat, 18 Jan 2020 19:35:45 -0800 Subject: [PATCH 19/20] Revert "GBA Memory: Improve gamepak prefetch timing" This reverts commit 8a6cf8dc9c00082e9917b4777f949de0d35ce710. --- CHANGES | 1 - src/gba/memory.c | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 255c5aca2..cb24f5ed0 100644 --- a/CHANGES +++ b/CHANGES @@ -92,7 +92,6 @@ Emulation fixes: - ARM: Fix stepping when events are pending - GBA DMA: Fix case where DMAs could get misaligned (fixes mgba.io/i/1092) - GBA Memory: Fix open bus from IWRAM (fixes mgba.io/i/1575) - - GBA Memory: Improve gamepak prefetch timing - GBA Timers: Improve timer startup emulation - GBA Video: Fix OpenGL renderer 512x512 backgrounds (fixes mgba.io/i/1572) - GBA Video: Fix BLDY for semitransparent sprite on non-target-2 backgrounds diff --git a/src/gba/memory.c b/src/gba/memory.c index 10cdd777c..091bfa1e1 100644 --- a/src/gba/memory.c +++ b/src/gba/memory.c @@ -1622,19 +1622,17 @@ int32_t GBAMemoryStall(struct ARMCore* cpu, int32_t wait) { maxLoads -= previousLoads; } - int32_t s = cpu->memory.activeSeqCycles16; + int32_t s = cpu->memory.activeSeqCycles16 + 1; int32_t n2s = cpu->memory.activeNonseqCycles16 - cpu->memory.activeSeqCycles16 + 1; // Figure out how many sequential loads we can jam in - int32_t stall = s + 1; + int32_t stall = s; int32_t loads = 1; while (stall < wait && loads < maxLoads) { stall += s; ++loads; } - memory->lastPrefetchedPc = cpu->gprs[ARM_PC] + WORD_SIZE_THUMB * (loads + previousLoads - 1); - if (stall > wait) { // The wait cannot take less time than the prefetch stalls wait = stall; @@ -1643,9 +1641,10 @@ int32_t GBAMemoryStall(struct ARMCore* cpu, int32_t wait) { // This instruction used to have an N, convert it to an S. wait -= n2s; - // The next |loads|S waitstates disappear entirely, so long as they're all in a row - wait -= stall - 1; + memory->lastPrefetchedPc = cpu->gprs[ARM_PC] + WORD_SIZE_THUMB * (loads + previousLoads - 1); + // The next |loads|S waitstates disappear entirely, so long as they're all in a row + cpu->cycles -= (s - 1) * loads; return wait; } From 5a543b2a6401313e73fa7efdca25320732cd3a63 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Sun, 19 Jan 2020 16:20:23 -0800 Subject: [PATCH 20/20] Switch: Fix build on libnx 3.0.0 --- include/mgba-util/platform/switch/threading.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mgba-util/platform/switch/threading.h b/include/mgba-util/platform/switch/threading.h index 2348b6983..76f1ae8b5 100644 --- a/include/mgba-util/platform/switch/threading.h +++ b/include/mgba-util/platform/switch/threading.h @@ -64,7 +64,7 @@ static inline int ThreadCreate(Thread* thread, ThreadEntry entry, void* context) if (!entry || !thread) { return 1; } - int res = threadCreate(thread, entry, context, 0x8000, 0x3B, 1); + int res = threadCreate(thread, entry, context, NULL, 0x8000, 0x3B, 1); if(R_FAILED(res)) { return res; }