- [Core] Added Blb file parsing [GI]

AssetStudio/AssetsManager.cs

@@ -432,19 +432,22 @@ namespace AssetStudio
             try
             {
                 using var stream = new OffsetStream(reader.BaseStream, 0);
-                if (AssetsHelper.TryGet(reader.FullPath, out var offsets))
+                foreach (var offset in stream.GetOffsets(reader.FullPath))
                 {
-                    foreach (var offset in offsets)
+                    var name = offset.ToString("X8");
+                    Logger.Info($"Loading Block {name}");
+
+                    var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), name);
+                    var subReader = new FileReader(dummyPath, stream, true);
+                    switch (subReader.FileType)
                     {
-                        LoadBlockSubFile(reader.FullPath, stream, offset);
+                        case FileType.BundleFile:
+                            LoadBundleFile(subReader, reader.FullPath, offset, false);
+                            break;
+                        case FileType.BlbFile:
+                            LoadBlbFile(subReader, reader.FullPath, offset, false);
+                            break;
                     }
-                }
-                else
-                {
-                    do
-                    {
-                        LoadBlockSubFile(reader.FullPath, stream, stream.AbsolutePosition);
-                    } while (stream.Remaining > 0);
                 }    
             }
             catch (Exception e)
@@ -456,16 +459,6 @@ namespace AssetStudio
                 reader.Dispose();
             }
         }
-        private void LoadBlockSubFile(string path, OffsetStream stream, long offset)
-        {
-            var name = offset.ToString("X8");
-            Logger.Info($"Loading Block {name}");
-
-            stream.Offset = offset;
-            var dummyPath = Path.Combine(Path.GetDirectoryName(path), name);
-            var subReader = new FileReader(dummyPath, stream, true);
-            LoadBundleFile(subReader, path, offset, false);
-        }
         private void LoadBlkFile(FileReader reader)
         {
             Logger.Info("Loading " + reader.FullPath);
@@ -546,6 +539,45 @@ namespace AssetStudio
                 reader.Dispose();
             }
         }
+        
+        private void LoadBlbFile(FileReader reader, string originalPath = null, long originalOffset = 0, bool log = true)
+        {
+            if (log)
+            {
+                Logger.Info("Loading " + reader.FullPath);
+            }
+            try
+            {
+                var blbFile = new BlbFile(reader, reader.FullPath);
+                foreach (var file in blbFile.fileList)
+                {
+                    var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName);
+                    var cabReader = new FileReader(dummyPath, file.stream);
+                    if (cabReader.FileType == FileType.AssetsFile)
+                    {
+                        LoadAssetsFromMemory(cabReader, originalPath ?? reader.FullPath, blbFile.m_Header.unityRevision, originalOffset);
+                    }
+                    else
+                    {
+                        Logger.Verbose("Caching resource stream");
+                        resourceFileReaders[file.fileName] = cabReader; //TODO
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                var str = $"Error while reading Blb file {reader.FullPath}";
+                if (originalPath != null)
+                {
+                    str += $" from {Path.GetFileName(originalPath)}";
+                }
+                Logger.Error(str, e);
+            }
+            finally
+            {
+                reader.Dispose();
+            }
+        }
 
         public void CheckStrippedVersion(SerializedFile assetsFile)
         {

AssetStudio/BlbFile.cs

@@ -0,0 +1,172 @@
+using System;
+using System.IO;
+using System.Linq;
+using K4os.Compression.LZ4;
+
+namespace AssetStudio
+{
+    public class BlbFile
+    {
+        private const uint DefaultUncompressedSize = 0x20000;
+
+        private BundleFile.StorageBlock[] m_BlocksInfo;
+        private BundleFile.Node[] m_DirectoryInfo;
+
+        public BundleFile.Header m_Header;
+        public StreamFile[] fileList;
+        public long Offset;
+
+
+        public BlbFile(FileReader reader, string path)
+        {
+            Offset = reader.Position;
+            reader.Endian = EndianType.LittleEndian;
+
+            var signature = reader.ReadStringToNull(4);
+            Logger.Verbose($"Parsed signature {signature}");
+            if (signature != "Blb\x02")
+                throw new Exception("not a Blb file");
+
+            var size = reader.ReadUInt32();
+            m_Header = new BundleFile.Header
+            {
+                version = 6,
+                unityVersion = "5.x.x",
+                unityRevision = "2017.4.30f1",
+                flags = (ArchiveFlags)0x43
+            };
+            m_Header.compressedBlocksInfoSize = size;
+            m_Header.uncompressedBlocksInfoSize = size;
+
+            Logger.Verbose($"Header: {m_Header}");
+
+            var header = reader.ReadBytes((int)m_Header.compressedBlocksInfoSize);
+            ReadBlocksInfoAndDirectory(header);
+            using var blocksStream = CreateBlocksStream(path);
+            ReadBlocks(reader, blocksStream);
+            ReadFiles(blocksStream, path);
+        }
+
+        private void ReadBlocksInfoAndDirectory(byte[] header)
+        {
+            using var stream = new MemoryStream(header);
+            using var reader = new EndianBinaryReader(stream, EndianType.LittleEndian);
+
+            m_Header.size = reader.ReadUInt32();
+            var lastUncompressedSize = reader.ReadUInt32();
+
+            reader.Position += 4;
+            var offset = reader.ReadInt64();
+            var compressionType = (CompressionType)reader.ReadByte();
+            var serializedFileVersion = (SerializedFileFormatVersion)reader.ReadByte();
+            reader.AlignStream();
+
+            var blocksInfoCount = reader.ReadInt32();
+            var nodesCount = reader.ReadInt32();
+
+            var blocksInfoOffset = reader.Position + reader.ReadInt64();
+            var nodesInfoOffset = reader.Position + reader.ReadInt64();
+            var bundleInfoOffset = reader.Position + reader.ReadInt64();
+
+            reader.Position = blocksInfoOffset;
+            m_BlocksInfo = new BundleFile.StorageBlock[blocksInfoCount];
+            Logger.Verbose($"Blocks count: {blocksInfoCount}");
+            for (int i = 0; i < blocksInfoCount; i++)
+            {
+                m_BlocksInfo[i] = new BundleFile.StorageBlock
+                {
+                    compressedSize = reader.ReadUInt32(),
+                    uncompressedSize = i == blocksInfoCount - 1 ? lastUncompressedSize : DefaultUncompressedSize,
+                    flags = (StorageBlockFlags)0x43
+                };
+
+                Logger.Verbose($"Block {i} Info: {m_BlocksInfo[i]}");
+            }
+
+            reader.Position = nodesInfoOffset;
+            m_DirectoryInfo = new BundleFile.Node[nodesCount];
+            Logger.Verbose($"Directory count: {nodesCount}");
+            for (int i = 0; i < nodesCount; i++)
+            {
+                m_DirectoryInfo[i] = new BundleFile.Node
+                {
+                    offset = reader.ReadInt32(),
+                    size = reader.ReadInt32()
+                };
+
+                var pathOffset = reader.Position + reader.ReadInt64();
+
+                var pos = reader.Position;
+                reader.Position = pathOffset;
+                m_DirectoryInfo[i].path = reader.ReadStringToNull();
+                reader.Position = pos;
+
+                Logger.Verbose($"Directory {i} Info: {m_DirectoryInfo[i]}");
+            }
+        }
+
+        private Stream CreateBlocksStream(string path)
+        {
+            Stream blocksStream;
+            var uncompressedSizeSum = (int)m_BlocksInfo.Sum(x => x.uncompressedSize);
+            Logger.Verbose($"Total size of decompressed blocks: 0x{uncompressedSizeSum:X8}");
+            if (uncompressedSizeSum >= int.MaxValue)
+                blocksStream = new FileStream(path + ".temp", FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
+            else
+                blocksStream = new MemoryStream(uncompressedSizeSum);
+            return blocksStream;
+        }
+
+        private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
+        {
+            foreach (var blockInfo in m_BlocksInfo)
+            {
+                var compressedSize = (int)blockInfo.compressedSize;
+                var uncompressedSize = (int)blockInfo.uncompressedSize;
+
+                var compressedBytes = BigArrayPool<byte>.Shared.Rent(compressedSize);
+                var uncompressedBytes = BigArrayPool<byte>.Shared.Rent(uncompressedSize);
+                reader.Read(compressedBytes, 0, compressedSize);
+
+                var compressedBytesSpan = compressedBytes.AsSpan(0, compressedSize);
+                var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize);
+
+                var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan);
+                if (numWrite != uncompressedSize)
+                {
+                    throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
+                }
+
+                blocksStream.Write(uncompressedBytes, 0, uncompressedSize);
+                BigArrayPool<byte>.Shared.Return(compressedBytes);
+                BigArrayPool<byte>.Shared.Return(uncompressedBytes);
+            }
+        }
+
+        private void ReadFiles(Stream blocksStream, string path)
+        {
+            Logger.Verbose($"Writing files from blocks stream...");
+
+            fileList = new StreamFile[m_DirectoryInfo.Length];
+            for (int i = 0; i < m_DirectoryInfo.Length; i++)
+            {
+                var node = m_DirectoryInfo[i];
+                var file = new StreamFile();
+                fileList[i] = file;
+                file.path = node.path;
+                file.fileName = Path.GetFileName(node.path);
+                if (node.size >= int.MaxValue)
+                {
+                    var extractPath = path + "_unpacked" + Path.DirectorySeparatorChar;
+                    Directory.CreateDirectory(extractPath);
+                    file.stream = new FileStream(extractPath + file.fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
+                }
+                else
+                    file.stream = new MemoryStream((int)node.size);
+                blocksStream.Position = node.offset;
+                blocksStream.CopyTo(file.stream, node.size);
+                file.stream.Position = 0;
+            }
+        }
+    }
+}

AssetStudio/BundleFile.cs

@@ -195,6 +195,9 @@ namespace AssetStudio
                         header.signature = "UnityFS";
                         goto case "UnityFS";
                     }
+                    header.version = reader.ReadUInt32();
+                    header.unityVersion = reader.ReadStringToNull();
+                    header.unityRevision = reader.ReadStringToNull();
                     break;
 
             }

AssetStudio/Crypto/BlkUtils.cs

@@ -10,7 +10,6 @@ namespace AssetStudio
         private const int DataOffset = 0x2A;
         private const int KeySize = 0x1000;
         private const int SeedBlockSize = 0x800;
-        private const int BufferSize = 0x10000;
 
         public static XORStream Decrypt(FileReader reader, Blk blk)
         {
@@ -64,48 +63,5 @@ namespace AssetStudio
 
             return new XORStream(reader.BaseStream, DataOffset, xorpad);
         }
-
-        public static IEnumerable<long> GetOffsets(this XORStream stream, string path)
-        {
-            if (AssetsHelper.TryGet(path, out var offsets))
-            {
-                foreach(var offset in offsets)
-                {
-                    stream.Offset = offset;
-                    yield return offset;
-                }
-            }
-            else
-            {
-                using var reader = new FileReader(path, stream, true);
-                var signature = reader.FileType switch
-                {
-                    FileType.BundleFile => "UnityFS\x00",
-                    FileType.Mhy0File => "mhy0",
-                    _ => throw new InvalidOperationException()
-                };
-
-                Logger.Verbose($"Prased signature: {signature}");
-
-                var signatureBytes = Encoding.UTF8.GetBytes(signature);
-                var buffer = BigArrayPool<byte>.Shared.Rent(BufferSize);
-                while (stream.Remaining > 0)
-                {
-                    var index = 0;
-                    var absOffset = stream.AbsolutePosition;
-                    var read = stream.Read(buffer);
-                    while (index < read)
-                    {
-                        index = buffer.AsSpan(0, read).Search(signatureBytes, index);
-                        if (index == -1) break;
-                        var offset = absOffset + index;
-                        stream.Offset = offset;
-                        yield return offset;
-                        index++;
-                    }
-                }
-                BigArrayPool<byte>.Shared.Return(buffer);
-            }
-        }
     }
 }

AssetStudio/FileReader.cs

@@ -16,6 +16,7 @@ namespace AssetStudio
         private static readonly byte[] zipMagic = { 0x50, 0x4B, 0x03, 0x04 };
         private static readonly byte[] zipSpannedMagic = { 0x50, 0x4B, 0x07, 0x08 };
         private static readonly byte[] mhy0Magic = { 0x6D, 0x68, 0x79, 0x30 };
+        private static readonly byte[] blbMagic = { 0x42, 0x6C, 0x62, 0x02 };
         private static readonly byte[] narakaMagic = { 0x15, 0x1E, 0x1C, 0x0D, 0x0D, 0x23, 0x21 };
         private static readonly byte[] gunfireMagic = { 0x7C, 0x6D, 0x79, 0x72, 0x27, 0x7A, 0x73, 0x78, 0x3F };
 
@@ -84,6 +85,11 @@ namespace AssetStudio
                             return FileType.Mhy0File;
                         }
                         Logger.Verbose($"Parsed signature does not match with expected signature {Convert.ToHexString(mhy0Magic)}");
+                        if (blbMagic.SequenceEqual(magic))
+                        {
+                            return FileType.BlbFile;
+                        }
+                        Logger.Verbose($"Parsed signature does not match with expected signature {Convert.ToHexString(mhy0Magic)}");
                         magic = ReadBytes(7);
                         Position = 0;
                         Logger.Verbose($"Parsed signature is {Convert.ToHexString(magic)}");
@@ -195,7 +201,7 @@ namespace AssetStudio
                         break;
                 }
             }
-            if (reader.FileType == FileType.BundleFile && game.Type.IsBlockFile())
+            if (reader.FileType == FileType.BundleFile && game.Type.IsBlockFile() || reader.FileType == FileType.BlbFile)
             {
                 Logger.Verbose("File might have multiple bundles !!");
                 try

AssetStudio/FileType.cs

@@ -17,6 +17,7 @@ namespace AssetStudio
         ZipFile,
         BlkFile,
         Mhy0File,
+        BlbFile,
         BlockFile
     }
 }

AssetStudio/OffsetStream.cs

@@ -1,10 +1,14 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Text;
 
 namespace AssetStudio
 {
     public class OffsetStream : Stream
     {
+        private const int BufferSize = 0x10000;
+
         private readonly Stream _baseStream;
         private long _offset;
 
@@ -64,5 +68,48 @@ namespace AssetStudio
         public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException();
         public override void SetLength(long value) => throw new NotImplementedException();
         public override void Flush() => throw new NotImplementedException();
+        public IEnumerable<long> GetOffsets(string path)
+        {
+            if (AssetsHelper.TryGet(path, out var offsets))
+            {
+                foreach (var offset in offsets)
+                {
+                    Offset = offset;
+                    yield return offset;
+                }
+            }
+            else
+            {
+                using var reader = new FileReader(path, this, true);
+                var signature = reader.FileType switch
+                {
+                    FileType.BundleFile => "UnityFS\x00",
+                    FileType.BlbFile => "Blb\x02",
+                    FileType.Mhy0File => "mhy0",
+                    _ => throw new InvalidOperationException()
+                };
+
+                Logger.Verbose($"Prased signature: {signature}");
+
+                var signatureBytes = Encoding.UTF8.GetBytes(signature);
+                var buffer = BigArrayPool<byte>.Shared.Rent(BufferSize);
+                while (Remaining > 0)
+                {
+                    var index = 0;
+                    var absOffset = AbsolutePosition;
+                    var read = Read(buffer);
+                    while (index < read)
+                    {
+                        index = buffer.AsSpan(0, read).Search(signatureBytes, index);
+                        if (index == -1) break;
+                        var offset = absOffset + index;
+                        Offset = offset;
+                        yield return offset;
+                        index++;
+                    }
+                }
+                BigArrayPool<byte>.Shared.Return(buffer);
+            }
+        }
     }
 }