From 5e5b6d11059cd5dd97e15e575bed965057e9c89d Mon Sep 17 00:00:00 2001 From: Razmoth <32140579+Razmoth@users.noreply.github.com> Date: Sat, 11 Nov 2023 20:12:16 +0400 Subject: [PATCH] - [Core] Remove `K4os.Compression.LZ4` --- AssetStudio.Utility/ShaderConverter.cs | 15 ++- AssetStudio/AssetStudio.csproj | 1 - AssetStudio/BlbFile.cs | 3 +- AssetStudio/BundleFile.cs | 13 ++- AssetStudio/Crypto/FairGuardUtils.cs | 69 ------------ AssetStudio/LZ4/LZ4Utils.cs | 143 +++++++++++++++++++++++++ AssetStudio/Mhy0File.cs | 5 +- 7 files changed, 165 insertions(+), 84 deletions(-) create mode 100644 AssetStudio/LZ4/LZ4Utils.cs diff --git a/AssetStudio.Utility/ShaderConverter.cs b/AssetStudio.Utility/ShaderConverter.cs index df46362..1e86b19 100644 --- a/AssetStudio.Utility/ShaderConverter.cs +++ b/AssetStudio.Utility/ShaderConverter.cs @@ -1,5 +1,4 @@ -using K4os.Compression.LZ4; -using SpirV; +using SpirV; using System; using System.Globalization; using System.IO; @@ -16,7 +15,11 @@ namespace AssetStudio if (shader.m_SubProgramBlob != null) //5.3 - 5.4 { var decompressedBytes = new byte[shader.decompressedSize]; - LZ4Codec.Decode(shader.m_SubProgramBlob, decompressedBytes); + var numWrite = LZ4.LZ4.Decompress(shader.m_SubProgramBlob, decompressedBytes); + if (numWrite != shader.decompressedSize) + { + throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {shader.decompressedSize} bytes"); + } using (var blobReader = new EndianBinaryReader(new MemoryStream(decompressedBytes), EndianType.LittleEndian)) { var program = new ShaderProgram(blobReader, shader); @@ -51,7 +54,11 @@ namespace AssetStudio } else { - LZ4Codec.Decode(shader.compressedBlob, (int)offset, (int)compressedLength, decompressedBytes, 0, (int)decompressedLength); + var numWrite = LZ4.LZ4.Decompress(shader.compressedBlob.AsSpan().Slice((int)offset, (int)compressedLength), decompressedBytes.AsSpan().Slice(0, (int)decompressedLength)); + if (numWrite != decompressedLength) + { + throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {decompressedLength} bytes"); + } } using (var blobReader = new EndianBinaryReader(new MemoryStream(decompressedBytes), EndianType.LittleEndian)) { diff --git a/AssetStudio/AssetStudio.csproj b/AssetStudio/AssetStudio.csproj index bf63a5e..03ef9c2 100644 --- a/AssetStudio/AssetStudio.csproj +++ b/AssetStudio/AssetStudio.csproj @@ -10,7 +10,6 @@ - diff --git a/AssetStudio/BlbFile.cs b/AssetStudio/BlbFile.cs index ff238c0..3d7bbb8 100644 --- a/AssetStudio/BlbFile.cs +++ b/AssetStudio/BlbFile.cs @@ -1,7 +1,6 @@ using System; using System.IO; using System.Linq; -using K4os.Compression.LZ4; namespace AssetStudio { @@ -131,7 +130,7 @@ namespace AssetStudio var compressedBytesSpan = compressedBytes.AsSpan(0, compressedSize); var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize); - var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan); + var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan, uncompressedBytesSpan); if (numWrite != uncompressedSize) { throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes"); diff --git a/AssetStudio/BundleFile.cs b/AssetStudio/BundleFile.cs index c59c897..65e1edd 100644 --- a/AssetStudio/BundleFile.cs +++ b/AssetStudio/BundleFile.cs @@ -1,5 +1,4 @@ -using K4os.Compression.LZ4; -using ZstdSharp; +using ZstdSharp; using System; using System.Data; using System.IO; @@ -438,7 +437,7 @@ namespace AssetStudio case CompressionType.Lz4HC: //LZ4HC { var uncompressedBytes = new byte[uncompressedSize]; - var numWrite = LZ4Codec.Decode(blocksInfoBytesSpan, uncompressedBytes); + var numWrite = LZ4.LZ4.Decompress(blocksInfoBytesSpan, uncompressedBytes); if (numWrite != uncompressedSize) { throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes"); @@ -554,7 +553,7 @@ namespace AssetStudio var uncompressedSize = (int)blockInfo.uncompressedSize; var uncompressedBytes = BigArrayPool.Shared.Rent(uncompressedSize); var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize); - var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan); + var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan, uncompressedBytesSpan); if (numWrite != uncompressedSize) { throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes"); @@ -578,7 +577,11 @@ namespace AssetStudio var uncompressedSize = (int)blockInfo.uncompressedSize; var uncompressedBytes = BigArrayPool.Shared.Rent(uncompressedSize); var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize); - FairGuardUtils.Lz4.Decompress(compressedBytesSpan, uncompressedBytesSpan); + var numWrite = LZ4.LZ4Inv.Decompress(compressedBytesSpan, uncompressedBytesSpan); + if (numWrite != uncompressedSize) + { + throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes"); + } blocksStream.Write(uncompressedBytes, 0, uncompressedSize); BigArrayPool.Shared.Return(compressedBytes); BigArrayPool.Shared.Return(uncompressedBytes); diff --git a/AssetStudio/Crypto/FairGuardUtils.cs b/AssetStudio/Crypto/FairGuardUtils.cs index b8fcbbe..23a53a5 100644 --- a/AssetStudio/Crypto/FairGuardUtils.cs +++ b/AssetStudio/Crypto/FairGuardUtils.cs @@ -232,74 +232,5 @@ namespace AssetStudio data[iteration] ^= (byte)(k - 0x61); } } - - public static class Lz4 - { - public static void Decompress(ReadOnlySpan cmp, Span dec) - { - int cmpPos = 0; - int decPos = 0; - - // ReSharper disable once VariableHidesOuterVariable - int GetLength(int length, ReadOnlySpan cmp) - { - byte sum; - - if (length == 0xf) - { - do - { - length += sum = cmp[cmpPos++]; - } while (sum == 0xff); - } - - return length; - } - - do - { - byte token = cmp[cmpPos++]; - - int encCount = (token >> 4) & 0xf; - int litCount = (token >> 0) & 0xf; - - //Copy literal chunk - litCount = GetLength(litCount, cmp); - - cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos)); - - cmpPos += litCount; - decPos += litCount; - - if (cmpPos >= cmp.Length) - { - break; - } - - //Copy compressed chunk - int back = cmp[cmpPos++] << 8 | - cmp[cmpPos++] << 0; - - encCount = GetLength(encCount, cmp) + 4; - - int encPos = decPos - back; - - if (encCount <= back) - { - dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos)); - - decPos += encCount; - } - else - { - while (encCount-- > 0) - { - dec[decPos++] = dec[encPos++]; - } - } - } while (cmpPos < cmp.Length && - decPos < dec.Length); - } - } } } \ No newline at end of file diff --git a/AssetStudio/LZ4/LZ4Utils.cs b/AssetStudio/LZ4/LZ4Utils.cs new file mode 100644 index 0000000..5c30269 --- /dev/null +++ b/AssetStudio/LZ4/LZ4Utils.cs @@ -0,0 +1,143 @@ +using System; + +namespace AssetStudio.LZ4; +public static class LZ4 +{ + public static int Decompress(ReadOnlySpan cmp, Span dec) + { + int cmpPos = 0; + int decPos = 0; + + // ReSharper disable once VariableHidesOuterVariable + int GetLength(int length, ReadOnlySpan cmp) + { + byte sum; + + if (length == 0xf) + { + do + { + length += sum = cmp[cmpPos++]; + } while (sum == 0xff); + } + + return length; + } + + do + { + byte token = cmp[cmpPos++]; + + int encCount = (token >> 0) & 0xf; + int litCount = (token >> 4) & 0xf; + + //Copy literal chunk + litCount = GetLength(litCount, cmp); + + cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos)); + + cmpPos += litCount; + decPos += litCount; + + if (cmpPos >= cmp.Length) + { + break; + } + + //Copy compressed chunk + int back = cmp[cmpPos++] << 0 | + cmp[cmpPos++] << 8; + + encCount = GetLength(encCount, cmp) + 4; + + int encPos = decPos - back; + + if (encCount <= back) + { + dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos)); + + decPos += encCount; + } + else + { + while (encCount-- > 0) + { + dec[decPos++] = dec[encPos++]; + } + } + } while (cmpPos < cmp.Length && + decPos < dec.Length); + + return decPos; + } +} +public static class LZ4Inv +{ + public static int Decompress(ReadOnlySpan cmp, Span dec) + { + int cmpPos = 0; + int decPos = 0; + + // ReSharper disable once VariableHidesOuterVariable + int GetLength(int length, ReadOnlySpan cmp) + { + byte sum; + + if (length == 0xf) + { + do + { + length += sum = cmp[cmpPos++]; + } while (sum == 0xff); + } + + return length; + } + + do + { + byte token = cmp[cmpPos++]; + + int encCount = (token >> 4) & 0xf; + int litCount = (token >> 0) & 0xf; + + //Copy literal chunk + litCount = GetLength(litCount, cmp); + + cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos)); + + cmpPos += litCount; + decPos += litCount; + + if (cmpPos >= cmp.Length) + { + break; + } + + //Copy compressed chunk + int back = cmp[cmpPos++] << 8 | + cmp[cmpPos++] << 0; + + encCount = GetLength(encCount, cmp) + 4; + + int encPos = decPos - back; + + if (encCount <= back) + { + dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos)); + + decPos += encCount; + } + else + { + while (encCount-- > 0) + { + dec[decPos++] = dec[encPos++]; + } + } + } while (cmpPos < cmp.Length && + decPos < dec.Length); + + return decPos; + } +} diff --git a/AssetStudio/Mhy0File.cs b/AssetStudio/Mhy0File.cs index bdf291f..4ea42d8 100644 --- a/AssetStudio/Mhy0File.cs +++ b/AssetStudio/Mhy0File.cs @@ -1,7 +1,6 @@ using System; using System.IO; using System.Linq; -using K4os.Compression.LZ4; namespace AssetStudio { @@ -55,7 +54,7 @@ namespace AssetStudio Logger.Verbose($"uncompressed blocksInfo size: 0x{m_Header.uncompressedBlocksInfoSize:X8}"); var compressedBlocksInfo = blocksInfoReader.ReadBytes((int)blocksInfoReader.Remaining); var uncompressedBlocksInfo = new byte[(int)m_Header.uncompressedBlocksInfoSize]; - var numWrite = LZ4Codec.Decode(compressedBlocksInfo, uncompressedBlocksInfo); + var numWrite = LZ4.LZ4.Decompress(compressedBlocksInfo, uncompressedBlocksInfo); if (numWrite != m_Header.uncompressedBlocksInfoSize) { throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {m_Header.uncompressedBlocksInfoSize} bytes"); @@ -128,7 +127,7 @@ namespace AssetStudio DescrambleEntry(compressedBytesSpan); Logger.Verbose($"Descrambled block signature {Convert.ToHexString(compressedBytes, 0, 4)}"); - var numWrite = LZ4Codec.Decode(compressedBytesSpan[0xC..compressedSize], uncompressedBytesSpan); + var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan[0xC..compressedSize], uncompressedBytesSpan); if (numWrite != uncompressedSize) { throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");