using System; using System.Buffers; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Text; using System.Threading; using static AssetStudio.ImportHelper; namespace AssetStudio { public class AssetsManager { public Game Game; public bool Silent = false; public bool SkipProcess = false; public bool ResolveDependencies = false; public string SpecifyUnityVersion; public CancellationTokenSource tokenSource = new CancellationTokenSource(); public List assetsFileList = new List(); public Dictionary ExportableTypes = new Dictionary(); internal Dictionary assetsFileIndexCache = new Dictionary(StringComparer.OrdinalIgnoreCase); internal Dictionary resourceFileReaders = new Dictionary(StringComparer.OrdinalIgnoreCase); internal List importFiles = new List(); internal HashSet importFilesHash = new HashSet(StringComparer.OrdinalIgnoreCase); internal HashSet noexistFiles = new HashSet(StringComparer.OrdinalIgnoreCase); internal HashSet assetsFileListHash = new HashSet(StringComparer.OrdinalIgnoreCase); public AssetsManager() { ExportableTypes.Add(ClassIDType.GameObject, true); ExportableTypes.Add(ClassIDType.Material, true); ExportableTypes.Add(ClassIDType.Texture2D, true); ExportableTypes.Add(ClassIDType.AudioClip, true); ExportableTypes.Add(ClassIDType.VideoClip, true); ExportableTypes.Add(ClassIDType.Mesh, false); ExportableTypes.Add(ClassIDType.Renderer, false); ExportableTypes.Add(ClassIDType.Shader, true); ExportableTypes.Add(ClassIDType.TextAsset, true); ExportableTypes.Add(ClassIDType.AnimationClip, true); ExportableTypes.Add(ClassIDType.MonoBehaviour, true); ExportableTypes.Add(ClassIDType.Font, true); ExportableTypes.Add(ClassIDType.Sprite, true); ExportableTypes.Add(ClassIDType.Animator, true); ExportableTypes.Add(ClassIDType.MiHoYoBinData, true); } public void LoadFiles(params string[] files) { if (Silent) { Logger.Silent = true; Progress.Silent = true; } var path = Path.GetDirectoryName(Path.GetFullPath(files[0])); MergeSplitAssets(path); var toReadFile = ProcessingSplitFiles(files.ToList()); Load(toReadFile); if (Silent) { Logger.Silent = false; Progress.Silent = false; } } public void LoadFolder(string path) { if (Silent) { Logger.Silent = true; Progress.Silent = true; } MergeSplitAssets(path, true); var files = Directory.GetFiles(path, "*.*", SearchOption.AllDirectories).ToList(); var toReadFile = ProcessingSplitFiles(files); Load(toReadFile); if (Silent) { Logger.Silent = false; Progress.Silent = false; } } private void Load(string[] files) { foreach (var file in files) { importFiles.Add(file); importFilesHash.Add(Path.GetFileName(file)); } Progress.Reset(); //use a for loop because list size can change for (var i = 0; i < importFiles.Count; i++) { LoadFile(importFiles[i]); Progress.Report(i + 1, importFiles.Count); if (tokenSource.IsCancellationRequested) { Logger.Info("Loading files has been aborted !!"); break; } if (!SkipProcess && !tokenSource.IsCancellationRequested) { ReadAssets(); ProcessAssets(); } } importFiles.Clear(); importFilesHash.Clear(); noexistFiles.Clear(); assetsFileListHash.Clear(); AssetsHelper.ClearOffsets(); //if (!SkipProcess && !tokenSource.IsCancellationRequested) //{ // ReadAssets(); // ProcessAssets(); //} } private void LoadFile(string fullName) { var reader = new FileReader(fullName); reader = reader.PreProcessing(Game); LoadFile(reader); } private void LoadFile(FileReader reader) { switch (reader.FileType) { case FileType.AssetsFile: LoadAssetsFile(reader); break; case FileType.BundleFile: LoadBundleFile(reader); break; case FileType.WebFile: LoadWebFile(reader); break; case FileType.GZipFile: LoadFile(DecompressGZip(reader)); break; case FileType.BrotliFile: LoadFile(DecompressBrotli(reader)); break; case FileType.ZipFile: LoadZipFile(reader); break; case FileType.BlockFile: LoadBlockFile(reader); break; case FileType.BlkFile: LoadBlkFile(reader); break; } } private void LoadAssetsFile(FileReader reader) { if (!assetsFileListHash.Contains(reader.FileName)) { Logger.Info($"Loading {reader.FullPath}"); try { var assetsFile = new SerializedFile(reader, this); CheckStrippedVersion(assetsFile); assetsFileList.Add(assetsFile); assetsFileListHash.Add(assetsFile.fileName); foreach (var sharedFile in assetsFile.m_Externals) { var sharedFileName = sharedFile.fileName; if (!importFilesHash.Contains(sharedFileName)) { var sharedFilePath = Path.Combine(Path.GetDirectoryName(reader.FullPath), sharedFileName); if (!noexistFiles.Contains(sharedFilePath)) { if (!File.Exists(sharedFilePath)) { var findFiles = Directory.GetFiles(Path.GetDirectoryName(reader.FullPath), sharedFileName, SearchOption.AllDirectories); if (findFiles.Length > 0) { sharedFilePath = findFiles[0]; } } if (File.Exists(sharedFilePath)) { importFiles.Add(sharedFilePath); importFilesHash.Add(sharedFileName); } else { noexistFiles.Add(sharedFilePath); } } } } } catch (Exception e) { Logger.Error($"Error while reading assets file {reader.FullPath}", e); reader.Dispose(); } } else { Logger.Info($"Skipping {reader.FullPath}"); reader.Dispose(); } } private void LoadAssetsFromMemory(FileReader reader, string originalPath, string unityVersion = null, long originalOffset = 0) { if (!assetsFileListHash.Contains(reader.FileName)) { try { var assetsFile = new SerializedFile(reader, this); assetsFile.originalPath = originalPath; assetsFile.offset = originalOffset; if (!string.IsNullOrEmpty(unityVersion) && assetsFile.header.m_Version < SerializedFileFormatVersion.Unknown_7) { assetsFile.SetVersion(unityVersion); } CheckStrippedVersion(assetsFile); assetsFileList.Add(assetsFile); assetsFileListHash.Add(assetsFile.fileName); //if (ResolveDependencies) //{ // foreach (var sharedFile in assetsFile.m_Externals) // { // var sharedFileName = sharedFile.fileName; // // if (!importFilesHash.Contains(sharedFileName)) // { // var sharedFilePath = Path.Combine(Path.GetDirectoryName(originalPath), sharedFileName); // if (!noexistFiles.Contains(sharedFilePath)) // { // if (AssetsHelper.TryAdd(sharedFileName, out var path)) // { // sharedFilePath = path; // } // if (File.Exists(sharedFilePath)) // { // if (!importFiles.Contains(sharedFilePath)) // { // importFiles.Add(sharedFilePath); // } // importFilesHash.Add(sharedFileName); // } // else // { // noexistFiles.Add(sharedFilePath); // } // } // } // } //} } catch (Exception e) { Logger.Error($"Error while reading assets file {reader.FullPath} from {Path.GetFileName(originalPath)}", e); resourceFileReaders.Add(reader.FileName, reader); } } else Logger.Info($"Skipping {originalPath} ({reader.FileName})"); } private void LoadBundleFile(FileReader reader, string originalPath = null, long originalOffset = 0) { Logger.Info("Loading " + reader.FullPath); try { var bundleFile = new BundleFile(reader, Game); foreach (var file in bundleFile.fileList) { var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName); var subReader = new FileReader(dummyPath, file.stream); if (subReader.FileType == FileType.AssetsFile) { LoadAssetsFromMemory(subReader, originalPath ?? reader.FullPath, bundleFile.m_Header.unityRevision, originalOffset); } else { resourceFileReaders[file.fileName] = subReader; //TODO } } } catch (InvalidCastException) { Logger.Error($"Game type mismatch, Expected {nameof(Mr0k)} but got {Game.Name} ({Game.GetType().Name}) !!"); } catch (Exception e) { var str = $"Error while reading bundle file {reader.FullPath}"; if (originalPath != null) { str += $" from {Path.GetFileName(originalPath)}"; } Logger.Error(str, e); } finally { reader.Dispose(); } } private void LoadWebFile(FileReader reader) { Logger.Info("Loading " + reader.FullPath); try { var webFile = new WebFile(reader); foreach (var file in webFile.fileList) { var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName); var subReader = new FileReader(dummyPath, file.stream); switch (subReader.FileType) { case FileType.AssetsFile: LoadAssetsFromMemory(subReader, reader.FullPath); break; case FileType.BundleFile: LoadBundleFile(subReader, reader.FullPath); break; case FileType.WebFile: LoadWebFile(subReader); break; case FileType.ResourceFile: resourceFileReaders[file.fileName] = subReader; //TODO break; } } } catch (Exception e) { Logger.Error($"Error while reading web file {reader.FullPath}", e); } finally { reader.Dispose(); } } private void LoadZipFile(FileReader reader) { Logger.Info("Loading " + reader.FileName); try { using (ZipArchive archive = new ZipArchive(reader.BaseStream, ZipArchiveMode.Read)) { List splitFiles = new List(); // register all files before parsing the assets so that the external references can be found // and find split files foreach (ZipArchiveEntry entry in archive.Entries) { if (entry.Name.Contains(".split")) { string baseName = Path.GetFileNameWithoutExtension(entry.Name); string basePath = Path.Combine(Path.GetDirectoryName(entry.FullName), baseName); if (!splitFiles.Contains(basePath)) { splitFiles.Add(basePath); importFilesHash.Add(baseName); } } else { importFilesHash.Add(entry.Name); } } // merge split files and load the result foreach (string basePath in splitFiles) { try { Stream splitStream = new MemoryStream(); int i = 0; while (true) { string path = $"{basePath}.split{i++}"; ZipArchiveEntry entry = archive.GetEntry(path); if (entry == null) break; using (Stream entryStream = entry.Open()) { entryStream.CopyTo(splitStream); } } splitStream.Seek(0, SeekOrigin.Begin); FileReader entryReader = new FileReader(basePath, splitStream); LoadFile(entryReader); } catch (Exception e) { Logger.Error($"Error while reading zip split file {basePath}", e); } } // load all entries foreach (ZipArchiveEntry entry in archive.Entries) { try { string dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), reader.FileName, entry.FullName); // create a new stream // - to store the deflated stream in // - to keep the data for later extraction Stream streamReader = new MemoryStream(); using (Stream entryStream = entry.Open()) { entryStream.CopyTo(streamReader); } streamReader.Position = 0; FileReader entryReader = new FileReader(dummyPath, streamReader); LoadFile(entryReader); if (entryReader.FileType == FileType.ResourceFile) { entryReader.Position = 0; if (!resourceFileReaders.ContainsKey(entry.Name)) { resourceFileReaders.Add(entry.Name, entryReader); } } } catch (Exception e) { Logger.Error($"Error while reading zip entry {entry.FullName}", e); } } } } catch (Exception e) { Logger.Error($"Error while reading zip file {reader.FileName}", e); } finally { reader.Dispose(); } } private void LoadBlockFile(FileReader reader) { Logger.Info("Loading " + reader.FullPath); try { using var stream = new BlockStream(reader.BaseStream, 0); if (AssetsHelper.TryGet(reader.FullPath, out var offsets)) { foreach (var offset in offsets) { stream.Offset = offset; var dummyPath = Path.Combine(reader.FileName, offset.ToString("X8")); var subReader = new FileReader(dummyPath, stream, true); LoadBundleFile(subReader, reader.FullPath, offset); } AssetsHelper.Remove(reader.FullPath); } else { do { stream.Offset = stream.RelativePosition; var dummyPath = Path.Combine(reader.FileName, stream.RelativePosition.ToString("X8")); var subReader = new FileReader(dummyPath, stream, true); LoadBundleFile(subReader, reader.FullPath, stream.RelativePosition); } while (stream.Remaining > 0); } } catch (Exception e) { Logger.Error($"Error while reading block file {reader.FileName}", e); } finally { reader.Dispose(); } } private void LoadBlkFile(FileReader reader) { Logger.Info("Loading " + reader.FullPath); try { using var stream = BlkUtils.Decrypt(reader, (Blk)Game); if (AssetsHelper.TryGet(reader.FullPath, out var offsets)) { foreach (var offset in offsets) { stream.Offset = offset; var dummyPath = Path.Combine(reader.FileName, offset.ToString("X8")); var subReader = new FileReader(dummyPath, stream, true); switch (subReader.FileType) { case FileType.BundleFile: LoadBundleFile(subReader, reader.FullPath, offset); break; case FileType.Mhy0File: LoadMhy0File(subReader, reader.FullPath, offset); break; } } AssetsHelper.Remove(reader.FullPath); } else { do { stream.Offset = stream.RelativePosition; var dummyPath = Path.Combine(reader.FileName, stream.RelativePosition.ToString("X8")); var subReader = new FileReader(dummyPath, stream, true); switch (subReader.FileType) { case FileType.BundleFile: LoadBundleFile(subReader, reader.FullPath, stream.RelativePosition); break; case FileType.Mhy0File: LoadMhy0File(subReader, reader.FullPath, stream.RelativePosition); break; } } while (stream.Remaining > 0); } } catch (InvalidCastException) { Logger.Error($"Game type mismatch, Expected {nameof(Blk)} but got {Game.Name} ({Game.GetType().Name}) !!"); } catch (Exception e) { Logger.Error($"Error while reading blk file {reader.FileName}", e); } finally { reader.Dispose(); } } private void LoadMhy0File(FileReader reader, string originalPath = null, long originalOffset = 0) { Logger.Info("Loading " + reader.FullPath); try { var mhy0File = new Mhy0File(reader, reader.FullPath, (Mhy0)Game); foreach (var file in mhy0File.fileList) { var dummyPath = Path.Combine(Path.GetDirectoryName(reader.FullPath), file.fileName); var cabReader = new FileReader(dummyPath, file.stream); if (cabReader.FileType == FileType.AssetsFile) { LoadAssetsFromMemory(cabReader, originalPath ?? reader.FullPath, mhy0File.m_Header.unityRevision, originalOffset); } else { resourceFileReaders[file.fileName] = cabReader; //TODO } } } catch (InvalidCastException) { Logger.Error($"Game type mismatch, Expected {nameof(Mhy0)} but got {Game.Name} ({Game.GetType().Name}) !!"); } catch (Exception e) { var str = $"Error while reading mhy0 file {reader.FullPath}"; if (originalPath != null) { str += $" from {Path.GetFileName(originalPath)}"; } Logger.Error(str, e); } finally { reader.Dispose(); } } public void CheckStrippedVersion(SerializedFile assetsFile) { if (assetsFile.IsVersionStripped && string.IsNullOrEmpty(SpecifyUnityVersion)) { throw new Exception("The Unity version has been stripped, please set the version in the options"); } if (!string.IsNullOrEmpty(SpecifyUnityVersion)) { assetsFile.SetVersion(SpecifyUnityVersion); } } public void Clear() { foreach (var assetsFile in assetsFileList) { assetsFile.Objects.Clear(); assetsFile.reader.Close(); } assetsFileList.Clear(); foreach (var resourceFileReader in resourceFileReaders) { resourceFileReader.Value.Close(); } resourceFileReaders.Clear(); assetsFileIndexCache.Clear(); tokenSource.Dispose(); tokenSource = new CancellationTokenSource(); GC.WaitForPendingFinalizers(); GC.Collect(); } private void ReadAssets() { Logger.Info("Read assets..."); var progressCount = assetsFileList.Sum(x => x.m_Objects.Count); int i = 0; Progress.Reset(); foreach (var assetsFile in assetsFileList) { foreach (var objectInfo in assetsFile.m_Objects) { if (tokenSource.IsCancellationRequested) { Logger.Info("Reading assets has been aborted !!"); return; } if (assetsFile.IsLoaded(objectInfo)) { continue; } var objectReader = new ObjectReader(assetsFile.reader, assetsFile, objectInfo, Game); try { Object obj; switch (objectReader.type) { case ClassIDType.Animation: obj = new Animation(objectReader); break; case ClassIDType.AnimationClip when ExportableTypes[ClassIDType.AnimationClip]: obj = new AnimationClip(objectReader); break; case ClassIDType.Animator when ExportableTypes[ClassIDType.Animator]: obj = new Animator(objectReader); break; case ClassIDType.AnimatorController: obj = new AnimatorController(objectReader); break; case ClassIDType.AnimatorOverrideController: obj = new AnimatorOverrideController(objectReader); break; case ClassIDType.AssetBundle: obj = new AssetBundle(objectReader); break; case ClassIDType.AudioClip when ExportableTypes[ClassIDType.AudioClip]: obj = new AudioClip(objectReader); break; case ClassIDType.Avatar: obj = new Avatar(objectReader); break; case ClassIDType.Font when ExportableTypes[ClassIDType.Font]: obj = new Font(objectReader); break; case ClassIDType.GameObject when ExportableTypes[ClassIDType.GameObject]: obj = new GameObject(objectReader); break; case ClassIDType.IndexObject when ExportableTypes[ClassIDType.MiHoYoBinData]: obj = new IndexObject(objectReader); break; case ClassIDType.Material when ExportableTypes[ClassIDType.Material]: obj = new Material(objectReader); break; case ClassIDType.Mesh when ExportableTypes[ClassIDType.Mesh]: obj = new Mesh(objectReader); break; case ClassIDType.MeshFilter: obj = new MeshFilter(objectReader); break; case ClassIDType.MeshRenderer when ExportableTypes[ClassIDType.Renderer]: obj = new MeshRenderer(objectReader); break; case ClassIDType.MiHoYoBinData when ExportableTypes[ClassIDType.MiHoYoBinData]: obj = new MiHoYoBinData(objectReader); break; case ClassIDType.MonoBehaviour when ExportableTypes[ClassIDType.MonoBehaviour]: obj = new MonoBehaviour(objectReader); break; case ClassIDType.MonoScript: obj = new MonoScript(objectReader); break; case ClassIDType.MovieTexture: obj = new MovieTexture(objectReader); break; case ClassIDType.PlayerSettings: obj = new PlayerSettings(objectReader); break; case ClassIDType.RectTransform: obj = new RectTransform(objectReader); break; case ClassIDType.Shader when ExportableTypes[ClassIDType.Shader]: obj = new Shader(objectReader); break; case ClassIDType.SkinnedMeshRenderer when ExportableTypes[ClassIDType.Renderer]: obj = new SkinnedMeshRenderer(objectReader); break; case ClassIDType.Sprite when ExportableTypes[ClassIDType.Sprite]: obj = new Sprite(objectReader); break; case ClassIDType.SpriteAtlas: obj = new SpriteAtlas(objectReader); break; case ClassIDType.TextAsset when ExportableTypes[ClassIDType.TextAsset]: obj = new TextAsset(objectReader); break; case ClassIDType.Texture2D when ExportableTypes[ClassIDType.Texture2D]: obj = new Texture2D(objectReader); break; case ClassIDType.Transform: obj = new Transform(objectReader); break; case ClassIDType.VideoClip when ExportableTypes[ClassIDType.VideoClip]: obj = new VideoClip(objectReader); break; case ClassIDType.ResourceManager: obj = new ResourceManager(objectReader); break; default: obj = new Object(objectReader); break; } assetsFile.AddObject(obj); } catch (Exception e) { var sb = new StringBuilder(); sb.AppendLine("Unable to load object") .AppendLine($"Assets {assetsFile.fileName}") .AppendLine($"Path {assetsFile.originalPath}") .AppendLine($"Type {objectReader.type}") .AppendLine($"PathID {objectInfo.m_PathID}") .Append(e); Logger.Error(sb.ToString()); } Progress.Report(++i, progressCount); } } } private void ProcessAssets() { if (tokenSource.IsCancellationRequested) return; Logger.Info("Process Assets..."); foreach (var assetsFile in assetsFileList) { foreach (var obj in assetsFile.Objects) { if (tokenSource.IsCancellationRequested) { Logger.Info("Processing assets has been aborted !!"); return; } if (obj is GameObject m_GameObject) { foreach (var pptr in m_GameObject.m_Components) { if (pptr.TryGet(out var m_Component)) { switch (m_Component) { case Transform m_Transform: m_GameObject.m_Transform = m_Transform; break; case MeshRenderer m_MeshRenderer: m_GameObject.m_MeshRenderer = m_MeshRenderer; break; case MeshFilter m_MeshFilter: m_GameObject.m_MeshFilter = m_MeshFilter; break; case SkinnedMeshRenderer m_SkinnedMeshRenderer: m_GameObject.m_SkinnedMeshRenderer = m_SkinnedMeshRenderer; break; case Animator m_Animator: m_GameObject.m_Animator = m_Animator; break; case Animation m_Animation: m_GameObject.m_Animation = m_Animation; break; } } } } else if (obj is SpriteAtlas m_SpriteAtlas) { if (m_SpriteAtlas.m_RenderDataMap.Count > 0) { foreach (var m_PackedSprite in m_SpriteAtlas.m_PackedSprites) { if (m_PackedSprite.TryGet(out var m_Sprite)) { if (m_Sprite.m_SpriteAtlas.IsNull) { m_Sprite.m_SpriteAtlas.Set(m_SpriteAtlas); } else { m_Sprite.m_SpriteAtlas.TryGet(out var m_SpriteAtlaOld); if (m_SpriteAtlaOld.m_IsVariant) { m_Sprite.m_SpriteAtlas.Set(m_SpriteAtlas); } } } } } } } } } } }