From b0e1458137d312019213a63ec169e27c51aefc57 Mon Sep 17 00:00:00 2001 From: YoshiRulz Date: Mon, 4 Oct 2021 10:35:48 +1000 Subject: [PATCH] Merge CRC32 impls and clean up --- src/BizHawk.Common/checksums/CRC32.cs | 180 +++++++++++++----- src/BizHawk.Common/checksums/SpecialCRC32.cs | 138 -------------- .../DiscHasher.cs | 9 +- .../Common/checksums/CRC32Tests.cs | 4 +- 4 files changed, 139 insertions(+), 192 deletions(-) delete mode 100644 src/BizHawk.Common/checksums/SpecialCRC32.cs diff --git a/src/BizHawk.Common/checksums/CRC32.cs b/src/BizHawk.Common/checksums/CRC32.cs index 7d7516a2b1..df0f111674 100644 --- a/src/BizHawk.Common/checksums/CRC32.cs +++ b/src/BizHawk.Common/checksums/CRC32.cs @@ -1,46 +1,134 @@ -using System; - -namespace BizHawk.Common -{ - // we could get a little list of crcs from here and make it clear which crc this class was for, and expose others - // http://www.ross.net/crc/download/crc_v3.txt - // TODO - why am I here? put me alongside hash_md5 and such - public static class CRC32 - { - // Lookup table for speed. - private static readonly uint[] Crc32Table; - - static CRC32() - { - Crc32Table = new uint[256]; - for (uint i = 0; i < 256; ++i) - { - uint crc = i; - for (int j = 8; j > 0; --j) - { - if ((crc & 1) == 1) - { - crc = (crc >> 1) ^ 0xEDB88320; - } - else - { - crc >>= 1; - } - } - - Crc32Table[i] = crc; - } - } - - public static uint Calculate(ReadOnlySpan data) - { - uint result = 0xFFFFFFFF; - foreach (var b in data) - { - result = (result >> 8) ^ Crc32Table[b ^ (result & 0xFF)]; - } - - return ~result; - } - } -} +using System; + +namespace BizHawk.Common +{ + /// Implementation of CRC-32 (i.e. POSIX cksum), intended for comparing discs against the Redump.org database + public sealed class CRC32 + { + /// coefficients of the polynomial, in the format Wikipedia calls "reversed" + public const uint POLYNOMIAL_CONST = 0xEDB88320U; + + private static readonly uint[] COMBINER_INIT_STATE; + + private static readonly uint[] CRC32Table; + + static CRC32() + { + // for Add (CRC32 computation): + CRC32Table = new uint[256]; + for (var i = 0U; i < 256U; i++) + { + var crc = i; + for (var j = 0; j < 8; j++) + { + var xor = (crc & 1U) == 1U; + crc >>= 1; + if (xor) crc ^= POLYNOMIAL_CONST; + } + CRC32Table[i] = crc; + } + + // for Incorporate: + var combinerState = (COMBINER_INIT_STATE = new uint[64]).AsSpan(); + var even = combinerState.Slice(start: 0, length: 32); // even-power-of-two zeros operator + var odd = combinerState.Slice(start: 32, length: 32); // odd-power-of-two zeros operator + // put operator for one zero bit in odd + odd[0] = POLYNOMIAL_CONST; + var oddTail = odd.Slice(1); + for (var n = 0; n < oddTail.Length; n++) oddTail[n] = 1U << n; + // put operator for two zero bits in even + gf2_matrix_square(even, odd); + // put operator for four zero bits in odd + gf2_matrix_square(odd, even); + } + + public static uint Calculate(ReadOnlySpan data) + { + CRC32 crc32 = new(); + crc32.Add(data); + return crc32.Result; + } + + private static void gf2_matrix_square(Span square, ReadOnlySpan mat) + { + if (mat.Length != square.Length) throw new ArgumentException(); + for (var n = 0; n < square.Length; n++) square[n] = gf2_matrix_times(mat, mat[n]); + } + + private static uint gf2_matrix_times(ReadOnlySpan mat, uint vec) + { + var matIdx = 0; + uint sum = 0U; + while (vec != 0U) + { + if ((vec & 1U) != 0U) sum ^= mat[matIdx]; + vec >>= 1; + matIdx++; + } + return sum; + } + + private uint _current = 0xFFFFFFFFU; + + /// The raw non-negated output + public uint Current + { + get => _current; + set => _current = value; + } + + /// The negated output (the typical result of the CRC calculation) + public uint Result => ~_current; + + public void Add(byte datum) + { + _current = CRC32Table[(_current ^ datum) & 0xFF] ^ (_current >> 8); + } + + public void Add(ReadOnlySpan data) + { + foreach (var b in data) + { +// Add(b); // I assume this would be slower + _current = CRC32Table[(_current ^ b) & 0xFF] ^ (_current >> 8); + } + } + + /// + /// Incorporates a pre-calculated CRC with the given length by combining crcs
+ /// It's a bit flaky, so be careful, but it works + /// + /// algorithm from zlib's crc32_combine. read http://www.leapsecond.com/tools/crcomb.c for more + public void Incorporate(uint crc, int len) + { + if (len == 0) return; // degenerate case + + Span combinerState = stackalloc uint[64]; + COMBINER_INIT_STATE.CopyTo(combinerState); + var even = combinerState.Slice(start: 0, length: 32); + var odd = combinerState.Slice(start: 32, length: 32); + + // apply len zeros to crc1 (first square will put the operator for one zero byte, eight zero bits, in even) + do + { + // apply zeros operator for this bit of len + gf2_matrix_square(even, odd); + if ((len & 1U) != 0U) _current = gf2_matrix_times(even, _current); + len >>= 1; + + // if no more bits set, then done + if (len == 0U) break; + + // another iteration of the loop with odd and even swapped + gf2_matrix_square(odd, even); + if ((len & 1U) != 0U) _current = gf2_matrix_times(odd, _current); + len >>= 1; + + // if no more bits set, then done + } while (len != 0U); + + // finally, combine and return + _current ^= crc; + } + } +} diff --git a/src/BizHawk.Common/checksums/SpecialCRC32.cs b/src/BizHawk.Common/checksums/SpecialCRC32.cs deleted file mode 100644 index 57a82d3ee6..0000000000 --- a/src/BizHawk.Common/checksums/SpecialCRC32.cs +++ /dev/null @@ -1,138 +0,0 @@ -#nullable disable - -using System; - -namespace BizHawk.Common -{ - /// - /// A stateful special CRC32 calculator - /// This may be absolutely standard and not special at all. I don't know, there were some differences between it and other CRC code I found in bizhawk - /// - public class SpecialCRC32 - { - private static readonly uint[] CRC32Table; - - static SpecialCRC32() - { - CRC32Table = new uint[256]; - for (uint i = 0; i < 256; ++i) - { - uint crc = i; - for (int j = 8; j > 0; --j) - { - if ((crc & 1) == 1) - crc = ((crc >> 1) ^ 0xEDB88320); - else - crc >>= 1; - } - CRC32Table[i] = crc; - } - } - - private uint current = 0xFFFFFFFF; - - public void Add(ReadOnlySpan data) - { - foreach (var b in data) current = CRC32Table[(current ^ b) & 0xFF] ^ (current >> 8); - } - - /// - /// The negated output (the typical result of the CRC calculation) - /// - public uint Result => current ^ 0xFFFFFFFF; - - /// - /// The raw non-negated output - /// - public uint Current - { - get => current; - set => current = value; - } - - private uint gf2_matrix_times(uint[] mat, uint vec) - { - int matIdx = 0; - uint sum = 0; - while (vec != 0) - { - if ((vec & 1) != 0) - sum ^= mat[matIdx]; - vec >>= 1; - matIdx++; - } - return sum; - } - - private void gf2_matrix_square(uint[] square, uint[] mat) - { - int n; - for (n = 0; n < 32; n++) - square[n] = gf2_matrix_times(mat, mat[n]); - } - - /// - /// Incorporates a pre-calculated CRC with the given length by combining crcs - /// It's a bit flaky, so be careful, but it works - /// - public void Incorporate(uint crc, int len) - { - current = crc32_combine(current, crc, len); - } - - //tables used by crc32_combine - private uint[] even, odd; - - //algorithm from zlib's crc32_combine. read http://www.leapsecond.com/tools/crcomb.c for more - private uint crc32_combine(uint crc1, uint crc2, int len2) - { - even ??= new uint[32]; // even-power-of-two zeros operator - odd ??= new uint[32]; // odd-power-of-two zeros operator - - // degenerate case - if (len2 == 0) - return crc1; - - // put operator for one zero bit in odd - odd[0] = 0xedb88320; //CRC-32 polynomial - uint row = 1; - for (int n = 1; n < 32; n++) - { - odd[n] = row; - row <<= 1; - } - - //put operator for two zero bits in even - gf2_matrix_square(even, odd); - - //put operator for four zero bits in odd - gf2_matrix_square(odd, even); - - //apply len2 zeros to crc1 (first square will put the operator for one zero byte, eight zero bits, in even) - do - { - //apply zeros operator for this bit of len2 - gf2_matrix_square(even, odd); - if ((len2 & 1) != 0) - crc1 = gf2_matrix_times(even, crc1); - len2 >>= 1; - - //if no more bits set, then done - if (len2 == 0) - break; - - //another iteration of the loop with odd and even swapped - gf2_matrix_square(odd, even); - if ((len2 & 1) != 0) - crc1 = gf2_matrix_times(odd, crc1); - len2 >>= 1; - - //if no more bits set, then done - } while (len2 != 0); - - //return combined crc - crc1 ^= crc2; - return crc1; - } - } -} diff --git a/src/BizHawk.Emulation.DiscSystem/DiscHasher.cs b/src/BizHawk.Emulation.DiscSystem/DiscHasher.cs index 022ee45dc5..8a108d40ed 100644 --- a/src/BizHawk.Emulation.DiscSystem/DiscHasher.cs +++ b/src/BizHawk.Emulation.DiscSystem/DiscHasher.cs @@ -24,10 +24,8 @@ namespace BizHawk.Emulation.DiscSystem //Tekken 3 (Europe) (Alt) and Tekken 3 (Europe) conflict in track 2 and 3 unfortunately, not sure what to do about this yet //the TOC isn't needed! //but it will help detect dumps with mangled TOCs which are all too common - // - //a possibly special CRC32 is used to help us match redump's DB elsewhere - SpecialCRC32 crc = new SpecialCRC32(); + CRC32 crc = new(); byte[] buffer2352 = new byte[2352]; var dsr = new DiscSectorReader(disc) @@ -36,7 +34,7 @@ namespace BizHawk.Emulation.DiscSystem }; //hash the TOC - static void AddAsBytesTo(SpecialCRC32 crc32, int i) + static void AddAsBytesTo(CRC32 crc32, int i) => crc32.Add(BitConverter.GetBytes(i)); AddAsBytesTo(crc, (int)disc.TOC.Session1Format); AddAsBytesTo(crc, disc.TOC.FirstRecordedTrackNumber); @@ -64,8 +62,7 @@ namespace BizHawk.Emulation.DiscSystem /// public uint Calculate_PSX_RedumpHash() { - //a special CRC32 is used to help us match redump's DB - SpecialCRC32 crc = new SpecialCRC32(); + CRC32 crc = new(); byte[] buffer2352 = new byte[2352]; var dsr = new DiscSectorReader(disc) diff --git a/src/BizHawk.Tests/Common/checksums/CRC32Tests.cs b/src/BizHawk.Tests/Common/checksums/CRC32Tests.cs index 7994719054..734c3d7912 100644 --- a/src/BizHawk.Tests/Common/checksums/CRC32Tests.cs +++ b/src/BizHawk.Tests/Common/checksums/CRC32Tests.cs @@ -33,12 +33,12 @@ namespace BizHawk.Tests.Common.checksums Assert.AreEqual(EXPECTED, CRC32.Calculate(data)); data = InitialiseArray(); - SpecialCRC32 crc32 = new(); + CRC32 crc32 = new(); crc32.Add(data); Assert.AreEqual(EXPECTED, crc32.Result); var dataExtra = InitialiseArrayExtra(); - SpecialCRC32 crc32Extra = new(); + CRC32 crc32Extra = new(); crc32Extra.Add(dataExtra); Assert.AreEqual(EXPECTED_EXTRA, crc32Extra.Result); crc32.Incorporate(crc32Extra.Result, dataExtra.Length);