From 051cb4f3e817acd6aff1e927f4f990e1e8f4a67d Mon Sep 17 00:00:00 2001 From: Chizuui Date: Thu, 27 Nov 2025 06:42:57 +0700 Subject: [PATCH] Endfield?? --- BeyondTools.SparkBuffer/BeanType.cs | 95 ++ .../BeyondTools.SparkBuffer.csproj | 16 + BeyondTools.SparkBuffer/EnumType.cs | 33 + .../Extensions/BinaryReaderExtensions.cs | 46 + .../Extensions/SparkTypeExtensions.cs | 8 + .../Properties/launchSettings.json | 8 + BeyondTools.SparkBuffer/SparkBufferDump.cs | 243 +++++ BeyondTools.SparkBuffer/SparkManager.cs | 38 + BeyondTools.SparkBuffer/SparkType.cs | 17 + BeyondTools.VFS/BeyondTools.VFS.csproj | 18 + BeyondTools.VFS/Crypto/CSChaCha20.cs | 930 ++++++++++++++++++ BeyondTools.VFS/EVFSBlockType.cs | 20 + BeyondTools.VFS/EVFSLoaderPosType.cs | 13 + .../Extensions/StreamExtensions.cs | 33 + BeyondTools.VFS/FVFBlockChunkInfo.cs | 16 + BeyondTools.VFS/FVFBlockFileInfo.cs | 17 + .../Properties/launchSettings.json | 8 + BeyondTools.VFS/VFBlockMainInfo.cs | 95 ++ BeyondTools.VFS/VFSDefine.cs | 11 + BeyondTools.VFS/VFSDump.cs | 103 ++ BeyondTools.sln | 28 + README.md | 19 +- 22 files changed, 1813 insertions(+), 2 deletions(-) create mode 100644 BeyondTools.SparkBuffer/BeanType.cs create mode 100644 BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj create mode 100644 BeyondTools.SparkBuffer/EnumType.cs create mode 100644 BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs create mode 100644 BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs create mode 100644 BeyondTools.SparkBuffer/Properties/launchSettings.json create mode 100644 BeyondTools.SparkBuffer/SparkBufferDump.cs create mode 100644 BeyondTools.SparkBuffer/SparkManager.cs create mode 100644 BeyondTools.SparkBuffer/SparkType.cs create mode 100644 BeyondTools.VFS/BeyondTools.VFS.csproj create mode 100644 BeyondTools.VFS/Crypto/CSChaCha20.cs create mode 100644 BeyondTools.VFS/EVFSBlockType.cs create mode 100644 BeyondTools.VFS/EVFSLoaderPosType.cs create mode 100644 BeyondTools.VFS/Extensions/StreamExtensions.cs create mode 100644 BeyondTools.VFS/FVFBlockChunkInfo.cs create mode 100644 BeyondTools.VFS/FVFBlockFileInfo.cs create mode 100644 BeyondTools.VFS/Properties/launchSettings.json create mode 100644 BeyondTools.VFS/VFBlockMainInfo.cs create mode 100644 BeyondTools.VFS/VFSDefine.cs create mode 100644 BeyondTools.VFS/VFSDump.cs create mode 100644 BeyondTools.sln diff --git a/BeyondTools.SparkBuffer/BeanType.cs b/BeyondTools.SparkBuffer/BeanType.cs new file mode 100644 index 0000000..c982564 --- /dev/null +++ b/BeyondTools.SparkBuffer/BeanType.cs @@ -0,0 +1,95 @@ +using BeyondTools.SparkBuffer.Extensions; + +namespace BeyondTools.SparkBuffer +{ + public struct BeanType + { + public int typeHash; + public string name; + public Field[] fields; + + public BeanType(BinaryReader reader) + { + typeHash = reader.ReadInt32(); + name = reader.ReadSparkBufferString(); + reader.Align4Bytes(); + var fieldCount = reader.ReadInt32(); + fields = new Field[fieldCount]; + + foreach (ref var field in fields.AsSpan()) + { + field.name = reader.ReadSparkBufferString(); + field.type = reader.ReadSparkType(); + switch (field.type) + { + case SparkType.Bool: + case SparkType.Byte: + case SparkType.Int: + case SparkType.Long: + case SparkType.Float: + case SparkType.Double: + case SparkType.String: + break; + case SparkType.Enum: + case SparkType.Bean: + reader.Align4Bytes(); + field.typeHash = reader.ReadInt32(); + break; + case SparkType.Array: + field.type2 = reader.ReadSparkType(); + + if (field.type2.Value.IsEnumOrBeanType()) + { + reader.Align4Bytes(); + field.typeHash = reader.ReadInt32(); + } + break; + case SparkType.Map: + field.type2 = reader.ReadSparkType(); + field.type3 = reader.ReadSparkType(); + + if (field.type2.Value.IsEnumOrBeanType()) + { + reader.Align4Bytes(); + field.typeHash = reader.ReadInt32(); + } + if (field.type3.Value.IsEnumOrBeanType()) + { + reader.Align4Bytes(); + field.typeHash2 = reader.ReadInt32(); + } + + break; + default: + throw new Exception(string.Format("Unsupported bean field type {0} at position {1}", field.type, reader.BaseStream.Position)); + } + } + } + + public struct Field + { + public string name; + public SparkType type; + + /// + /// or key type + /// + public SparkType? type2; + + /// + /// value type + /// + public SparkType? type3; + + /// + /// , , , or key type hash + /// + public int? typeHash; + + /// + /// value type hash + /// + public int? typeHash2; + } + } +} diff --git a/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj b/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj new file mode 100644 index 0000000..88f11e2 --- /dev/null +++ b/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj @@ -0,0 +1,16 @@ + + + + Exe + net9.0 + enable + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/BeyondTools.SparkBuffer/EnumType.cs b/BeyondTools.SparkBuffer/EnumType.cs new file mode 100644 index 0000000..5b35559 --- /dev/null +++ b/BeyondTools.SparkBuffer/EnumType.cs @@ -0,0 +1,33 @@ +using BeyondTools.SparkBuffer.Extensions; + +namespace BeyondTools.SparkBuffer +{ + public struct EnumType + { + public int typeHash; + public string name; + public EnumItem[] enums; + + public EnumType(BinaryReader reader) + { + typeHash = reader.ReadInt32(); + name = reader.ReadSparkBufferString(); + reader.Align4Bytes(); + var enumCount = reader.ReadInt32(); + enums = new EnumItem[enumCount]; + + foreach (ref var enumItem in enums.AsSpan()) + { + enumItem.name = reader.ReadSparkBufferString(); + reader.Align4Bytes(); + enumItem.value = reader.ReadInt32(); + } + } + + public struct EnumItem + { + public string name; + public int value; + } + } +} diff --git a/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs b/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs new file mode 100644 index 0000000..4d460af --- /dev/null +++ b/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs @@ -0,0 +1,46 @@ +using System.IO; +using System.Text; + +namespace BeyondTools.SparkBuffer.Extensions +{ + public static class BinaryReaderExtensions + { + public static SparkType ReadSparkType(this BinaryReader reader) + => (SparkType)reader.ReadByte(); + + public static long Seek(this BinaryReader reader, long pos, SeekOrigin seekOrigin = SeekOrigin.Begin) + => reader.BaseStream.Seek(pos, seekOrigin); + + public static string ReadSparkBufferString(this BinaryReader reader) + { + using MemoryStream buffer = new(); + while (true) + { + byte b = reader.ReadByte(); + if (b == 0) + break; + buffer.WriteByte(b); + } + + return Encoding.UTF8.GetString(buffer.ToArray()); + } + + public static string ReadSparkBufferStringOffset(this BinaryReader reader) + { + var stringOffset = reader.ReadInt32(); + if (stringOffset == -1) + return string.Empty; + + var oldPosition = reader.BaseStream.Position; + + reader.Seek(stringOffset); + var tmp = reader.ReadSparkBufferString(); + + reader.BaseStream.Position = oldPosition; + return tmp; + } + + public static void Align4Bytes(this BinaryReader reader) + => reader.Seek((reader.BaseStream.Position - 1) + (4 - ((reader.BaseStream.Position - 1) % 4))); + } +} diff --git a/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs b/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs new file mode 100644 index 0000000..e0fe3ad --- /dev/null +++ b/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs @@ -0,0 +1,8 @@ +namespace BeyondTools.SparkBuffer.Extensions +{ + public static class SparkTypeExtensions + { + public static bool IsEnumOrBeanType(this SparkType type) + => type is SparkType.Enum or SparkType.Bean; + } +} diff --git a/BeyondTools.SparkBuffer/Properties/launchSettings.json b/BeyondTools.SparkBuffer/Properties/launchSettings.json new file mode 100644 index 0000000..fb10990 --- /dev/null +++ b/BeyondTools.SparkBuffer/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "BeyondTools.SparkBuffer": { + "commandName": "Project", + "commandLineArgs": "C:\\EndfieldOut\\Assets\\TextAsset\\TableCfg C:\\EndfieldOut\\TableCfgOutput" + } + } +} \ No newline at end of file diff --git a/BeyondTools.SparkBuffer/SparkBufferDump.cs b/BeyondTools.SparkBuffer/SparkBufferDump.cs new file mode 100644 index 0000000..fb92251 --- /dev/null +++ b/BeyondTools.SparkBuffer/SparkBufferDump.cs @@ -0,0 +1,243 @@ +using BeyondTools.SparkBuffer.Extensions; +using ConsoleAppFramework; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace BeyondTools.SparkBuffer +{ + internal class SparkBufferDump + { + static void Main(string[] args) + { + ConsoleApp.Run(args, ( + [Argument] string tableCfgDir, + [Argument] string outputDir) => + { + if (!Directory.Exists(tableCfgDir)) + throw new FileNotFoundException($"{tableCfgDir} isn't a valid directory"); + if (!Directory.Exists(outputDir)) + Directory.CreateDirectory(outputDir); + + var success = 0; + var processingCount = 0; + foreach (var tablePath in Directory.EnumerateFiles(tableCfgDir)) + { + var fileName = Path.GetFileName(tablePath); + + Console.WriteLine("Reading {0}...", fileName); + processingCount++; + + using var file = File.OpenRead(tablePath); + using var binaryReader = new BinaryReader(file); + + try + { + var typeDefOffset = binaryReader.ReadInt32(); + var rootDefOffset = binaryReader.ReadInt32(); + var dataOffset = binaryReader.ReadInt32(); + + binaryReader.Seek(typeDefOffset); + SparkManager.ReadTypeDefinitions(binaryReader); + + binaryReader.Seek(rootDefOffset); + var rootDef = new BeanType.Field + { + type = binaryReader.ReadSparkType(), + name = binaryReader.ReadSparkBufferString() + }; + + if (rootDef.type.IsEnumOrBeanType()) + { + binaryReader.Align4Bytes(); + rootDef.typeHash = binaryReader.ReadInt32(); + } + if (rootDef.type == SparkType.Map) + { + rootDef.type2 = binaryReader.ReadSparkType(); + rootDef.type3 = binaryReader.ReadSparkType(); + + if (rootDef.type2.Value.IsEnumOrBeanType()) + { + binaryReader.Align4Bytes(); + rootDef.typeHash = binaryReader.ReadInt32(); + } + if (rootDef.type3.Value.IsEnumOrBeanType()) + { + binaryReader.Align4Bytes(); + rootDef.typeHash2 = binaryReader.ReadInt32(); + } + } + + binaryReader.Seek(dataOffset); + var resultFilePath = Path.Combine(outputDir, $"{rootDef.name}.json"); + switch (rootDef.type) + { + case SparkType.Bean: + var rootBeanType = SparkManager.BeanTypeFromHash((int)rootDef.typeHash!); + var beanDump = ReadBeanAsJObject(binaryReader, rootBeanType); + File.WriteAllText(resultFilePath, beanDump!.ToString()); + break; + case SparkType.Map: + var mapDump = ReadMapAsJObject(binaryReader, rootDef); + File.WriteAllText(resultFilePath, JsonSerializer.Serialize(mapDump, SparkManager.jsonSerializerOptions)); + break; + default: + throw new NotSupportedException(string.Format("Unsupported root type {0}", rootDef.type)); + } + + Console.WriteLine("Dumped {0} successfully", rootDef.name); + success++; + } + catch (Exception ex) + { + Console.WriteLine("Error in reading {0}, Error: {1}", fileName, ex.ToString()); + } + } + + Console.WriteLine("Dumped {0}/{1}", success, processingCount); + }); + } + + static JsonObject? ReadMapAsJObject(BinaryReader binaryReader, BeanType.Field typeDef) + { + var mapDump = new JsonObject(); + var kvCount = binaryReader.ReadInt32(); + + for (int i = 0; i < kvCount; i++) + { + var key = typeDef.type2 switch + { + SparkType.String => binaryReader.ReadSparkBufferStringOffset(), + SparkType.Int => binaryReader.ReadInt32().ToString(), + _ => throw new NotSupportedException(string.Format("Unsupported map key type {0}", typeDef.type2)), + }; + mapDump[key] = null; + } + + for (int i = 0; i < kvCount; i++) + { + mapDump[i] = typeDef.type3 switch + { + SparkType.Bean => ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)typeDef.typeHash2!), true), + SparkType.String => binaryReader.ReadSparkBufferStringOffset(), + SparkType.Int => binaryReader.ReadInt32(), + _ => throw new NotSupportedException(string.Format("Unsupported map value type {0}", typeDef.type3)), + }; + } + + return mapDump; + } + + static JsonObject? ReadBeanAsJObject(BinaryReader binaryReader, BeanType beanType, bool pointer = false) + { + long? pointerOrigin = null; + if (pointer) + { + var beanOffset = binaryReader.ReadInt32(); + if (beanOffset == -1) + return null; + + pointerOrigin = binaryReader.BaseStream.Position; + binaryReader.Seek(beanOffset); + } + + var dumpObj = new JsonObject(); + + foreach (var (fieldIndex, beanField) in beanType.fields.Index()) + { + long? origin = null; + if (beanField.type == SparkType.Array) + { + var fieldOffset = binaryReader.ReadInt32(); + if (fieldOffset == -1) + { + dumpObj[beanField.name] = null; + continue; + } + + origin = binaryReader.BaseStream.Position; + binaryReader.Seek(fieldOffset); + } + + switch (beanField.type) + { + case SparkType.Array: + var jArray = new JsonArray(); + + var itemCount = binaryReader.ReadInt32(); + while (itemCount-- > 0) + { + switch (beanField.type2) + { + case SparkType.String: + jArray.Add(binaryReader.ReadSparkBufferStringOffset()); + break; + case SparkType.Bean: + jArray.Add(ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true)); + break; + case SparkType.Float: + jArray.Add(binaryReader.ReadSingle()); + break; + case SparkType.Long: + jArray.Add(binaryReader.ReadInt64()); + break; + case SparkType.Int: + case SparkType.Enum: + jArray.Add(binaryReader.ReadInt32()); + break; + case SparkType.Bool: + jArray.Add(binaryReader.ReadBoolean()); + break; + default: + throw new NotSupportedException(string.Format("Unsupported array type {0} on bean field, position: {1}", beanField.type2, binaryReader.BaseStream.Position)); + } + } + + dumpObj[beanField.name] = jArray; + break; + case SparkType.Int: + case SparkType.Enum: + dumpObj[beanField.name] = binaryReader.ReadInt32(); + break; + case SparkType.Long: + dumpObj[beanField.name] = binaryReader.ReadInt64(); + break; + case SparkType.Float: + dumpObj[beanField.name] = binaryReader.ReadSingle(); + break; + case SparkType.Double: + dumpObj[beanField.name] = binaryReader.ReadDouble(); + break; + case SparkType.String: + dumpObj[beanField.name] = binaryReader.ReadSparkBufferStringOffset(); + break; + case SparkType.Bean: + dumpObj[beanField.name] = ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true); + break; + case SparkType.Bool: + dumpObj[beanField.name] = binaryReader.ReadBoolean(); + if (beanType.fields.Length > fieldIndex + 1 && beanType.fields[fieldIndex + 1].type != SparkType.Bool) + binaryReader.Align4Bytes(); + break; + case SparkType.Map: + var mapOffset = binaryReader.ReadInt32(); + var mapOrigin = binaryReader.BaseStream.Position; + binaryReader.Seek(mapOffset); + dumpObj[beanField.name] = ReadMapAsJObject(binaryReader, beanField); + binaryReader.Seek(mapOrigin); + break; + case SparkType.Byte: + throw new Exception(string.Format("Dumping bean field type {0} isn't supported, position: {1}", beanField.type, binaryReader.BaseStream.Position)); + } + + if (origin is not null) + binaryReader.BaseStream.Position = (long)origin; + } + + if (pointerOrigin is not null) + binaryReader.BaseStream.Position = (long)pointerOrigin; + + return dumpObj; + } + } +} diff --git a/BeyondTools.SparkBuffer/SparkManager.cs b/BeyondTools.SparkBuffer/SparkManager.cs new file mode 100644 index 0000000..91d46ef --- /dev/null +++ b/BeyondTools.SparkBuffer/SparkManager.cs @@ -0,0 +1,38 @@ +using BeyondTools.SparkBuffer.Extensions; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace BeyondTools.SparkBuffer +{ + public static class SparkManager + { + public static readonly JsonSerializerOptions jsonSerializerOptions = new() { IncludeFields = true, WriteIndented = true, NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals }; + + private static readonly Dictionary beanTypeMap = []; + private static readonly Dictionary enumTypeMap = []; + + public static BeanType BeanTypeFromHash(int hash) + => beanTypeMap[hash]; + + public static void ReadTypeDefinitions(BinaryReader reader) + { + var typeDefCount = reader.ReadInt32(); + while (typeDefCount-- > 0) + { + var sparkType = reader.ReadSparkType(); + reader.Align4Bytes(); + + if (sparkType == SparkType.Enum) + { + var enumType = new EnumType(reader); + enumTypeMap.TryAdd(enumType.typeHash, enumType); + } + else if (sparkType == SparkType.Bean) + { + var beanType = new BeanType(reader); + beanTypeMap.TryAdd(beanType.typeHash, beanType); + } + } + } + } +} diff --git a/BeyondTools.SparkBuffer/SparkType.cs b/BeyondTools.SparkBuffer/SparkType.cs new file mode 100644 index 0000000..56fcd1c --- /dev/null +++ b/BeyondTools.SparkBuffer/SparkType.cs @@ -0,0 +1,17 @@ +namespace BeyondTools.SparkBuffer +{ + public enum SparkType : byte + { + Bool, + Byte, + Int, + Long, + Float, + Double, + Enum, + String, + Bean, + Array, + Map + } +} diff --git a/BeyondTools.VFS/BeyondTools.VFS.csproj b/BeyondTools.VFS/BeyondTools.VFS.csproj new file mode 100644 index 0000000..e60ba34 --- /dev/null +++ b/BeyondTools.VFS/BeyondTools.VFS.csproj @@ -0,0 +1,18 @@ + + + + Exe + net9.0 + enable + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + diff --git a/BeyondTools.VFS/Crypto/CSChaCha20.cs b/BeyondTools.VFS/Crypto/CSChaCha20.cs new file mode 100644 index 0000000..6415cd2 --- /dev/null +++ b/BeyondTools.VFS/Crypto/CSChaCha20.cs @@ -0,0 +1,930 @@ +/* + * Copyright (c) 2015, 2018 Scott Bennett + * (c) 2018-2023 Kaarlo Räihä + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +using System; +using System.IO; +using System.Threading.Tasks; +using System.Runtime.Intrinsics; +using System.Runtime.CompilerServices; + +namespace BeyondTools.VFS.Crypto; + +/// +/// Chosen SIMD mode +/// +public enum SimdMode +{ + /// + /// Autodetect + /// + AutoDetect = 0, + + /// + /// 128 bit SIMD + /// + V128, + + /// + /// 256 bit SIMD + /// + V256, + + /// + /// 512 bit SIMD + /// + V512, + + /// + /// No SIMD + /// + None +} + +/// +/// Class for ChaCha20 encryption / decryption +/// +public sealed class CSChaCha20 : IDisposable +{ + /// + /// Only allowed key lenght in bytes + /// + public const int allowedKeyLength = 32; + + /// + /// Only allowed nonce lenght in bytes + /// + public const int allowedNonceLength = 12; + + /// + /// How many bytes are processed per loop + /// + public const int processBytesAtTime = 64; + + private const int stateLength = 16; + + /// + /// The ChaCha20 state (aka "context") + /// + private readonly uint[] state = new uint[stateLength]; + + /// + /// Determines if the objects in this class have been disposed of. Set to true by the Dispose() method. + /// + private bool isDisposed = false; + + /// + /// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens. + /// + /// + /// See ChaCha20 Spec Section 2.4 for a detailed description of the inputs. + /// + /// + /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers + /// + /// + /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers + /// + /// + /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer + /// + public CSChaCha20(byte[] key, byte[] nonce, uint counter) + { + KeySetup(key); + IvSetup(nonce, counter); + } + + /// + /// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens. + /// + /// + /// See ChaCha20 Spec Section 2.4 for a detailed description of the inputs. + /// + /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers + /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers + /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian unsigned integer + public CSChaCha20(ReadOnlySpan key, ReadOnlySpan nonce, uint counter) + { + KeySetup(key.ToArray()); + IvSetup(nonce.ToArray(), counter); + } + + /// + /// The ChaCha20 state (aka "context"). Read-Only. + /// + public uint[] State + { + get + { + return state; + } + } + + + // These are the same constants defined in the reference implementation. + // http://cr.yp.to/streamciphers/timings/estreambench/submissions/salsa20/chacha8/ref/chacha.c + private static readonly byte[] sigma = "expand 32-byte k"u8.ToArray(); + private static readonly byte[] tau = "expand 16-byte k"u8.ToArray(); + + /// + /// Set up the ChaCha state with the given key. A 32-byte key is required and enforced. + /// + /// + /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers + /// + private void KeySetup(byte[] key) + { + if (key == null) + { + throw new ArgumentNullException("Key is null"); + } + + if (key.Length != allowedKeyLength) + { + throw new ArgumentException($"Key length must be {allowedKeyLength}. Actual: {key.Length}"); + } + + state[4] = Util.U8To32Little(key, 0); + state[5] = Util.U8To32Little(key, 4); + state[6] = Util.U8To32Little(key, 8); + state[7] = Util.U8To32Little(key, 12); + + byte[] constants = key.Length == allowedKeyLength ? sigma : tau; + int keyIndex = key.Length - 16; + + state[8] = Util.U8To32Little(key, keyIndex + 0); + state[9] = Util.U8To32Little(key, keyIndex + 4); + state[10] = Util.U8To32Little(key, keyIndex + 8); + state[11] = Util.U8To32Little(key, keyIndex + 12); + + state[0] = Util.U8To32Little(constants, 0); + state[1] = Util.U8To32Little(constants, 4); + state[2] = Util.U8To32Little(constants, 8); + state[3] = Util.U8To32Little(constants, 12); + } + + /// + /// Set up the ChaCha state with the given nonce (aka Initialization Vector or IV) and block counter. A 12-byte nonce and a 4-byte counter are required. + /// + /// + /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers + /// + /// + /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer + /// + private void IvSetup(byte[] nonce, uint counter) + { + if (nonce == null) + { + // There has already been some state set up. Clear it before exiting. + Dispose(); + throw new ArgumentNullException("Nonce is null"); + } + + if (nonce.Length != allowedNonceLength) + { + // There has already been some state set up. Clear it before exiting. + Dispose(); + throw new ArgumentException($"Nonce length must be {allowedNonceLength}. Actual: {nonce.Length}"); + } + + state[12] = counter; + state[13] = Util.U8To32Little(nonce, 0); + state[14] = Util.U8To32Little(nonce, 4); + state[15] = Util.U8To32Little(nonce, 8); + } + + private static SimdMode DetectSimdMode() + { + if (Vector512.IsHardwareAccelerated) + { + return SimdMode.V512; + } + else if (Vector256.IsHardwareAccelerated) + { + return SimdMode.V256; + } + else if (Vector128.IsHardwareAccelerated) + { + return SimdMode.V128; + } + + return SimdMode.None; + } + + #region Encryption methods + + /// + /// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Output byte array, must have enough bytes + /// Input byte array + /// Number of bytes to encrypt + /// Chosen SIMD mode (default is auto-detect) + public void EncryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect) + { + if (output == null) + { + throw new ArgumentNullException("output", "Output cannot be null"); + } + + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (numBytes < 0 || numBytes > input.Length) + { + throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]"); + } + + if (output.Length < numBytes) + { + throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkBytes(output, input, numBytes, simdMode); + } + + /// + /// Encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output) + /// + /// Output stream + /// Input stream + /// How many bytes to read and write at time, default is 1024 + /// Chosen SIMD mode (default is auto-detect) + public void EncryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect) + { + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime); + } + + /// + /// Async encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output) + /// + /// Output stream + /// Input stream + /// How many bytes to read and write at time, default is 1024 + /// Chosen SIMD mode (default is auto-detect) + public async Task EncryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect) + { + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime); + } + + /// + /// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Output byte array, must have enough bytes + /// Input byte array + /// Chosen SIMD mode (default is auto-detect) + public void EncryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (output == null) + { + throw new ArgumentNullException("output", "Output cannot be null"); + } + + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkBytes(output, input, input.Length, simdMode); + } + + /// + /// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Input byte array + /// Number of bytes to encrypt + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains encrypted bytes + public byte[] EncryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (numBytes < 0 || numBytes > input.Length) + { + throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] returnArray = new byte[numBytes]; + WorkBytes(returnArray, input, numBytes, simdMode); + return returnArray; + } + + /// + /// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Input byte array + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains encrypted bytes + public byte[] EncryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] returnArray = new byte[input.Length]; + WorkBytes(returnArray, input, input.Length, simdMode); + return returnArray; + } + + /// + /// Encrypt string as UTF8 byte array, returns byte array that is allocated by method. + /// + /// Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform + /// Input string + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains encrypted bytes + public byte[] EncryptString(string input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] utf8Bytes = System.Text.Encoding.UTF8.GetBytes(input); + byte[] returnArray = new byte[utf8Bytes.Length]; + + WorkBytes(returnArray, utf8Bytes, utf8Bytes.Length, simdMode); + return returnArray; + } + + #endregion // Encryption methods + + + #region // Decryption methods + + /// + /// Decrypt arbitrary-length byte array (input), writing the resulting byte array to the output buffer. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Output byte array + /// Input byte array + /// Number of bytes to decrypt + /// Chosen SIMD mode (default is auto-detect) + public void DecryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect) + { + if (output == null) + { + throw new ArgumentNullException("output", "Output cannot be null"); + } + + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (numBytes < 0 || numBytes > input.Length) + { + throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]"); + } + + if (output.Length < numBytes) + { + throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkBytes(output, input, numBytes, simdMode); + } + + /// + /// Decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output) + /// + /// Output stream + /// Input stream + /// How many bytes to read and write at time, default is 1024 + /// Chosen SIMD mode (default is auto-detect) + public void DecryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect) + { + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime); + } + + /// + /// Async decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output) + /// + /// Output stream + /// Input stream + /// How many bytes to read and write at time, default is 1024 + /// Chosen SIMD mode (default is auto-detect) + public async Task DecryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect) + { + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime); + } + + /// + /// Decrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Output byte array, must have enough bytes + /// Input byte array + /// Chosen SIMD mode (default is auto-detect) + public void DecryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (output == null) + { + throw new ArgumentNullException("output", "Output cannot be null"); + } + + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + WorkBytes(output, input, input.Length, simdMode); + } + + /// + /// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Input byte array + /// Number of bytes to encrypt + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains decrypted bytes + public byte[] DecryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (numBytes < 0 || numBytes > input.Length) + { + throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] returnArray = new byte[numBytes]; + WorkBytes(returnArray, input, numBytes, simdMode); + return returnArray; + } + + /// + /// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method. + /// + /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method + /// Input byte array + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains decrypted bytes + public byte[] DecryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] returnArray = new byte[input.Length]; + WorkBytes(returnArray, input, input.Length, simdMode); + return returnArray; + } + + /// + /// Decrypt UTF8 byte array to string. + /// + /// Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform + /// Byte array + /// Chosen SIMD mode (default is auto-detect) + /// Byte array that contains encrypted bytes + public string DecryptUTF8ByteArray(byte[] input, SimdMode simdMode = SimdMode.AutoDetect) + { + if (input == null) + { + throw new ArgumentNullException("input", "Input cannot be null"); + } + + if (simdMode == SimdMode.AutoDetect) + { + simdMode = DetectSimdMode(); + } + + byte[] tempArray = new byte[input.Length]; + + WorkBytes(tempArray, input, input.Length, simdMode); + return System.Text.Encoding.UTF8.GetString(tempArray); + } + + #endregion // Decryption methods + + private void WorkStreams(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024) + { + int readBytes; + + byte[] inputBuffer = new byte[howManyBytesToProcessAtTime]; + byte[] outputBuffer = new byte[howManyBytesToProcessAtTime]; + + while ((readBytes = input.Read(inputBuffer, 0, howManyBytesToProcessAtTime)) > 0) + { + // Encrypt or decrypt + WorkBytes(output: outputBuffer, input: inputBuffer, numBytes: readBytes, simdMode); + + // Write buffer + output.Write(outputBuffer, 0, readBytes); + } + } + + private async Task WorkStreamsAsync(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024) + { + byte[] readBytesBuffer = new byte[howManyBytesToProcessAtTime]; + byte[] writeBytesBuffer = new byte[howManyBytesToProcessAtTime]; + int howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime); + + while (howManyBytesWereRead > 0) + { + // Encrypt or decrypt + WorkBytes(output: writeBytesBuffer, input: readBytesBuffer, numBytes: howManyBytesWereRead, simdMode); + + // Write + await output.WriteAsync(writeBytesBuffer, 0, howManyBytesWereRead); + + // Read more + howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime); + } + } + + /// + /// Encrypt or decrypt an arbitrary-length byte array (input), writing the resulting byte array to the output buffer. The number of bytes to read from the input buffer is determined by numBytes. + /// + /// Output byte array + /// Input byte array + /// How many bytes to process + /// Chosen SIMD mode (default is auto-detect) + private void WorkBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode) + { + if (isDisposed) + { + throw new ObjectDisposedException("state", "The ChaCha state has been disposed"); + } + + uint[] x = new uint[stateLength]; // Working buffer + byte[] tmp = new byte[processBytesAtTime]; // Temporary buffer + int offset = 0; + + int howManyFullLoops = numBytes / processBytesAtTime; + int tailByteCount = numBytes - howManyFullLoops * processBytesAtTime; + + for (int loop = 0; loop < howManyFullLoops; loop++) + { + UpdateStateAndGenerateTemporaryBuffer(state, x, tmp); + + if (simdMode == SimdMode.V512) + { + // 1 x 64 bytes + Vector512 inputV = Vector512.Create(input, offset); + Vector512 tmpV = Vector512.Create(tmp, 0); + Vector512 outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset); + } + else if (simdMode == SimdMode.V256) + { + // 2 x 32 bytes + Vector256 inputV = Vector256.Create(input, offset); + Vector256 tmpV = Vector256.Create(tmp, 0); + Vector256 outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset); + + inputV = Vector256.Create(input, offset + 32); + tmpV = Vector256.Create(tmp, 32); + outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset + 32); + } + else if (simdMode == SimdMode.V128) + { + // 4 x 16 bytes + Vector128 inputV = Vector128.Create(input, offset); + Vector128 tmpV = Vector128.Create(tmp, 0); + Vector128 outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset); + + inputV = Vector128.Create(input, offset + 16); + tmpV = Vector128.Create(tmp, 16); + outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset + 16); + + inputV = Vector128.Create(input, offset + 32); + tmpV = Vector128.Create(tmp, 32); + outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset + 32); + + inputV = Vector128.Create(input, offset + 48); + tmpV = Vector128.Create(tmp, 48); + outputV = inputV ^ tmpV; + outputV.CopyTo(output, offset + 48); + } + else + { + for (int i = 0; i < processBytesAtTime; i += 4) + { + // Small unroll + int start = i + offset; + output[start] = (byte)(input[start] ^ tmp[i]); + output[start + 1] = (byte)(input[start + 1] ^ tmp[i + 1]); + output[start + 2] = (byte)(input[start + 2] ^ tmp[i + 2]); + output[start + 3] = (byte)(input[start + 3] ^ tmp[i + 3]); + } + } + + offset += processBytesAtTime; + } + + // In case there are some bytes left + if (tailByteCount > 0) + { + UpdateStateAndGenerateTemporaryBuffer(state, x, tmp); + + for (int i = 0; i < tailByteCount; i++) + { + output[i + offset] = (byte)(input[i + offset] ^ tmp[i]); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void UpdateStateAndGenerateTemporaryBuffer(uint[] stateToModify, uint[] workingBuffer, byte[] temporaryBuffer) + { + // Copy state to working buffer + Buffer.BlockCopy(stateToModify, 0, workingBuffer, 0, stateLength * sizeof(uint)); + + for (int i = 0; i < 10; i++) + { + QuarterRound(workingBuffer, 0, 4, 8, 12); + QuarterRound(workingBuffer, 1, 5, 9, 13); + QuarterRound(workingBuffer, 2, 6, 10, 14); + QuarterRound(workingBuffer, 3, 7, 11, 15); + + QuarterRound(workingBuffer, 0, 5, 10, 15); + QuarterRound(workingBuffer, 1, 6, 11, 12); + QuarterRound(workingBuffer, 2, 7, 8, 13); + QuarterRound(workingBuffer, 3, 4, 9, 14); + } + + for (int i = 0; i < stateLength; i++) + { + Util.ToBytes(temporaryBuffer, Util.Add(workingBuffer[i], stateToModify[i]), 4 * i); + } + + stateToModify[12] = Util.AddOne(stateToModify[12]); + if (stateToModify[12] <= 0) + { + /* Stopping at 2^70 bytes per nonce is the user's responsibility */ + stateToModify[13] = Util.AddOne(stateToModify[13]); + } + } + + /// + /// The ChaCha Quarter Round operation. It operates on four 32-bit unsigned integers within the given buffer at indices a, b, c, and d. + /// + /// + /// The ChaCha state does not have four integer numbers: it has 16. So the quarter-round operation works on only four of them -- hence the name. Each quarter round operates on four predetermined numbers in the ChaCha state. + /// See ChaCha20 Spec Sections 2.1 - 2.2. + /// + /// A ChaCha state (vector). Must contain 16 elements. + /// Index of the first number + /// Index of the second number + /// Index of the third number + /// Index of the fourth number + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void QuarterRound(uint[] x, uint a, uint b, uint c, uint d) + { + x[a] = Util.Add(x[a], x[b]); + x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 16); + + x[c] = Util.Add(x[c], x[d]); + x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 12); + + x[a] = Util.Add(x[a], x[b]); + x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 8); + + x[c] = Util.Add(x[c], x[d]); + x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 7); + } + + #region Destructor and Disposer + + /// + /// Clear and dispose of the internal state. The finalizer is only called if Dispose() was never called on this cipher. + /// + ~CSChaCha20() + { + Dispose(false); + } + + /// + /// Clear and dispose of the internal state. Also request the GC not to call the finalizer, because all cleanup has been taken care of. + /// + public void Dispose() + { + Dispose(true); + /* + * The Garbage Collector does not need to invoke the finalizer because Dispose(bool) has already done all the cleanup needed. + */ + GC.SuppressFinalize(this); + } + + /// + /// This method should only be invoked from Dispose() or the finalizer. This handles the actual cleanup of the resources. + /// + /// + /// Should be true if called by Dispose(); false if called by the finalizer + /// + private void Dispose(bool disposing) + { + if (!isDisposed) + { + if (disposing) + { + /* Cleanup managed objects by calling their Dispose() methods */ + } + + /* Cleanup any unmanaged objects here */ + Array.Clear(state, 0, stateLength); + } + + isDisposed = true; + } + + #endregion // Destructor and Disposer +} + +/// +/// Utilities that are used during compression +/// +public static class Util +{ + /// + /// n-bit left rotation operation (towards the high bits) for 32-bit integers. + /// + /// + /// + /// The result of (v LEFTSHIFT c) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint Rotate(uint v, int c) + { + unchecked + { + return v << c | v >> 32 - c; + } + } + + /// + /// Unchecked integer exclusive or (XOR) operation. + /// + /// + /// + /// The result of (v XOR w) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint XOr(uint v, uint w) + { + return unchecked(v ^ w); + } + + /// + /// Unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32. + /// + /// + /// See ChaCha20 Spec Section 2.1. + /// + /// + /// + /// The result of (v + w) modulo 2^32 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint Add(uint v, uint w) + { + return unchecked(v + w); + } + + /// + /// Add 1 to the input parameter using unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32. + /// + /// + /// See ChaCha20 Spec Section 2.1. + /// + /// + /// The result of (v + 1) modulo 2^32 + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint AddOne(uint v) + { + return unchecked(v + 1); + } + + /// + /// Convert four bytes of the input buffer into an unsigned 32-bit integer, beginning at the inputOffset. + /// + /// + /// + /// An unsigned 32-bit integer + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint U8To32Little(byte[] p, int inputOffset) + { + unchecked + { + return p[inputOffset] + | (uint)p[inputOffset + 1] << 8 + | (uint)p[inputOffset + 2] << 16 + | (uint)p[inputOffset + 3] << 24; + } + } + + /// + /// Serialize the input integer into the output buffer. The input integer will be split into 4 bytes and put into four sequential places in the output buffer, starting at the outputOffset. + /// + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void ToBytes(byte[] output, uint input, int outputOffset) + { + unchecked + { + output[outputOffset] = (byte)input; + output[outputOffset + 1] = (byte)(input >> 8); + output[outputOffset + 2] = (byte)(input >> 16); + output[outputOffset + 3] = (byte)(input >> 24); + } + } +} \ No newline at end of file diff --git a/BeyondTools.VFS/EVFSBlockType.cs b/BeyondTools.VFS/EVFSBlockType.cs new file mode 100644 index 0000000..d2f5db1 --- /dev/null +++ b/BeyondTools.VFS/EVFSBlockType.cs @@ -0,0 +1,20 @@ +namespace BeyondTools.VFS +{ + public enum EVFSBlockType : byte + { + All, + + InitialAudio = 1, + InitialBundle, + BundleManifest, + LowShader, + Audio = 11, + Bundle, + TextAsset = 14, + Video, + IV, + Streaming, + IFixPatch = 21, + Raw = 31 + } +} diff --git a/BeyondTools.VFS/EVFSLoaderPosType.cs b/BeyondTools.VFS/EVFSLoaderPosType.cs new file mode 100644 index 0000000..cbd70bb --- /dev/null +++ b/BeyondTools.VFS/EVFSLoaderPosType.cs @@ -0,0 +1,13 @@ +namespace BeyondTools.VFS +{ + public enum EVFSLoaderPosType : byte + { + None, + PersistAsset, + StreamAsset, + VFS = 10, + VFS_PersistAsset, + VFS_StreamAsset, + VFS_Build + } +} diff --git a/BeyondTools.VFS/Extensions/StreamExtensions.cs b/BeyondTools.VFS/Extensions/StreamExtensions.cs new file mode 100644 index 0000000..dc321b5 --- /dev/null +++ b/BeyondTools.VFS/Extensions/StreamExtensions.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BeyondTools.VFS.Extensions +{ + internal static class StreamExtensions + { + public static void CopyBytes(this Stream inStream, Stream outStream, long? count = null) + { + if (count == null) + { + inStream.CopyTo(outStream); + return; + } + + long readBytes = 0L; + var buffer = new byte[64 * 1024]; + do + { + var toRead = Math.Min((long)(count - readBytes), buffer.LongLength); + var readNow = inStream.Read(buffer, 0, (int)toRead); + if (readNow == 0) + break; + outStream.Write(buffer, 0, readNow); + readBytes += readNow; + } while (readBytes < count); + } + } +} diff --git a/BeyondTools.VFS/FVFBlockChunkInfo.cs b/BeyondTools.VFS/FVFBlockChunkInfo.cs new file mode 100644 index 0000000..237fb4d --- /dev/null +++ b/BeyondTools.VFS/FVFBlockChunkInfo.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; + +namespace BeyondTools.VFS +{ + public struct FVFBlockChunkInfo + { + public const string FILE_EXTENSION = ".chk"; + + public UInt128 md5Name; + public UInt128 contentMD5; + public long length; + public EVFSBlockType blockType; + [JsonIgnore] + public FVFBlockFileInfo[] files; + } +} diff --git a/BeyondTools.VFS/FVFBlockFileInfo.cs b/BeyondTools.VFS/FVFBlockFileInfo.cs new file mode 100644 index 0000000..a404321 --- /dev/null +++ b/BeyondTools.VFS/FVFBlockFileInfo.cs @@ -0,0 +1,17 @@ +namespace BeyondTools.VFS +{ + public struct FVFBlockFileInfo + { + public string fileName; + public long fileNameHash; + public UInt128 fileChunkMD5Name; + public UInt128 fileDataMD5; + public long offset; + public long len; + public EVFSBlockType blockType; + public bool bUseEncrypt; + public long ivSeed; + public bool bIsDirect; + public EVFSLoaderPosType loaderPosType; + } +} diff --git a/BeyondTools.VFS/Properties/launchSettings.json b/BeyondTools.VFS/Properties/launchSettings.json new file mode 100644 index 0000000..5ca7bc3 --- /dev/null +++ b/BeyondTools.VFS/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "BeyondTools.VFS": { + "commandName": "Project", + "commandLineArgs": "C:/Users/rafi1/Desktop/etc/Beyond_TMP/Game/Beyond_Data/StreamingAssets InitialBundle C:/EndfieldOut/Assets" + } + } +} \ No newline at end of file diff --git a/BeyondTools.VFS/VFBlockMainInfo.cs b/BeyondTools.VFS/VFBlockMainInfo.cs new file mode 100644 index 0000000..8ea4225 --- /dev/null +++ b/BeyondTools.VFS/VFBlockMainInfo.cs @@ -0,0 +1,95 @@ +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using System.Text; + +namespace BeyondTools.VFS +{ + public class VFBlockMainInfo + { + public VFBlockMainInfo(byte[] bytes, int offset = 0) + { + version = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(int); + // TODO: CRC and stuff idk + offset += 12; + + ushort groupCfgNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(ushort); + groupCfgName = Encoding.UTF8.GetString(bytes.AsSpan(offset, groupCfgNameLength)); + offset += groupCfgNameLength; + + groupCfgHashName = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + groupFileInfoNum = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(int); + + groupChunksLength = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + blockType = (EVFSBlockType)bytes[offset++]; + + var chunkCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); + allChunks = GC.AllocateUninitializedArray(chunkCount); + offset += sizeof(int); + + foreach (ref var chunk in allChunks.AsSpan()) + { + chunk.md5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); + offset += Marshal.SizeOf(); + + chunk.contentMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); + offset += Marshal.SizeOf(); + + chunk.length = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + chunk.blockType = (EVFSBlockType)bytes[offset++]; + + var fileCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset)); + chunk.files = GC.AllocateUninitializedArray(fileCount); + offset += sizeof(int); + + foreach (ref var file in chunk.files.AsSpan()) + { + ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(ushort); + file.fileName = Encoding.UTF8.GetString(bytes.AsSpan(offset, fileNameLength)); + offset += fileNameLength; + + file.fileNameHash = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + file.fileChunkMD5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); + offset += Marshal.SizeOf(); + + file.fileDataMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset)); + offset += Marshal.SizeOf(); + + file.offset = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + file.len = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + + file.blockType = (EVFSBlockType)bytes[offset++]; + file.bUseEncrypt = Convert.ToBoolean(bytes[offset++]); + if (file.bUseEncrypt) + { + file.ivSeed = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset)); + offset += sizeof(long); + } + } + } + + } + + public int version; + public string groupCfgName; + public long groupCfgHashName; + public int groupFileInfoNum; + public long groupChunksLength; + public EVFSBlockType blockType; + public FVFBlockChunkInfo[] allChunks; + } +} diff --git a/BeyondTools.VFS/VFSDefine.cs b/BeyondTools.VFS/VFSDefine.cs new file mode 100644 index 0000000..81f0314 --- /dev/null +++ b/BeyondTools.VFS/VFSDefine.cs @@ -0,0 +1,11 @@ +namespace BeyondTools.VFS +{ + public static class VFSDefine + { + public const string CHACHA_KEY = "eU1cu+MYQiaYdVherRzV86pv/N/lIU/9gIk+5n5Vj4Y="; + public const string VFS_DIR = "VFS"; + public const int VFS_PROTO_VERSION = 3; + public const int VFS_VFB_HEAD_LEN = 16; + public const int BLOCK_HEAD_LEN = 12; + } +} diff --git a/BeyondTools.VFS/VFSDump.cs b/BeyondTools.VFS/VFSDump.cs new file mode 100644 index 0000000..fe3d893 --- /dev/null +++ b/BeyondTools.VFS/VFSDump.cs @@ -0,0 +1,103 @@ +using BeyondTools.VFS.Crypto; +using BeyondTools.VFS.Extensions; +using ConsoleAppFramework; +using System.IO.Hashing; +using System.Text; + +namespace BeyondTools.VFS; +internal class VFSDump +{ + static readonly Dictionary blockTypeMap = new() + { + { EVFSBlockType.Audio, "MainAudio" }, + { EVFSBlockType.Bundle, "MainBundles" }, + { EVFSBlockType.BundleManifest, "BundleManifest" }, + { EVFSBlockType.IFixPatch, "IFixPatchOut" }, + { EVFSBlockType.InitialAudio, "InitAudio" }, + { EVFSBlockType.InitialBundle, "InitBundles" }, + { EVFSBlockType.IV, "IV" }, + { EVFSBlockType.LowShader, "LowShader" }, + // { EVFSBlockType.Raw, "" }, Not present + { EVFSBlockType.Streaming, "Streaming" }, + { EVFSBlockType.TextAsset, "TextAsset" }, + { EVFSBlockType.Video, "Video" }, + }; + + static void Main(string[] args) + { + ConsoleApp.Run(args, ( + [Argument] string streamingAssetsPath, + [Argument] EVFSBlockType dumpAssetType = EVFSBlockType.All, + [Argument] string? outputDir = null) => + { + streamingAssetsPath = Path.Combine(streamingAssetsPath, VFSDefine.VFS_DIR); + outputDir ??= Path.Combine(AppContext.BaseDirectory, "Assets"); + if (dumpAssetType == EVFSBlockType.All) + { + foreach (var type in blockTypeMap.Keys) + { + DumpAssetByType(streamingAssetsPath, type, outputDir); + } + } + else + { + DumpAssetByType(streamingAssetsPath, dumpAssetType, outputDir); + } + }); + } + + private static void DumpAssetByType(string streamingAssetsPath, EVFSBlockType dumpAssetType, string outputDir) + { + Console.WriteLine("Dumping {0} files...", dumpAssetType.ToString()); + + // TODO: This is a temporary solution that makes this thing only worked on some EVFSBlockType, i haven't been able to figure out Crc32Utils.UnityCRC64 + var blockDir = Directory.EnumerateDirectories(streamingAssetsPath).First(x => x.Split('/', '\\').Last().StartsWith(Convert.ToHexString(Crc32.Hash(Encoding.UTF8.GetBytes(blockTypeMap[dumpAssetType]))))); + var blockFilePath = Path.Combine(blockDir, blockDir.Split('/', '\\').Last() + ".blc"); + + var blockFile = File.ReadAllBytes(blockFilePath); + byte[] nonce = GC.AllocateUninitializedArray(VFSDefine.BLOCK_HEAD_LEN); + Buffer.BlockCopy(blockFile, 0, nonce, 0, nonce.Length); + + var chacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), nonce, 1); + var decryptedBytes = chacha.DecryptBytes(blockFile[VFSDefine.BLOCK_HEAD_LEN..]); + Buffer.BlockCopy(decryptedBytes, 0, blockFile, VFSDefine.BLOCK_HEAD_LEN, decryptedBytes.Length); + + var vfBlockMainInfo = new VFBlockMainInfo(blockFile); + foreach (var chunk in vfBlockMainInfo.allChunks) + { + var chunkMd5Name = Convert.ToHexString(BitConverter.GetBytes(chunk.md5Name)) + FVFBlockChunkInfo.FILE_EXTENSION; + var chunkFs = File.OpenRead(Path.Join(blockDir, chunkMd5Name)); + foreach (var file in chunk.files) + { + var filePath = Path.Combine(outputDir, file.fileName); + if (!Directory.Exists(Path.GetDirectoryName(filePath))) + Directory.CreateDirectory(Path.GetDirectoryName(filePath) ?? throw new InvalidDataException($"Cannot get directory name of {filePath}")); + + if (file.bUseEncrypt) + { + byte[] fileNonce = GC.AllocateUninitializedArray(VFSDefine.BLOCK_HEAD_LEN); + Buffer.BlockCopy(BitConverter.GetBytes(vfBlockMainInfo.version), 0, fileNonce, 0, sizeof(int)); + Buffer.BlockCopy(BitConverter.GetBytes(file.ivSeed), 0, fileNonce, sizeof(int), sizeof(long)); + + var fileChacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), fileNonce, 1); + var encryptedMs = new MemoryStream(); + chunkFs.CopyBytes(encryptedMs, file.len); + + // TODO: Should've used stream decryptor for better perf, but idk how to get it working + File.WriteAllBytes(filePath, fileChacha.DecryptBytes(encryptedMs.ToArray())); + } + else + { + var fileFs = File.OpenWrite(filePath); + chunkFs.Seek(file.offset, SeekOrigin.Begin); + chunkFs.CopyBytes(fileFs, file.len); + fileFs.Dispose(); + } + + } + + Console.WriteLine("Dumped {0} file(s) from chunk {1}", chunk.files.Length, chunkMd5Name); + chunkFs.Dispose(); + } + } +} \ No newline at end of file diff --git a/BeyondTools.sln b/BeyondTools.sln new file mode 100644 index 0000000..c9c0a34 --- /dev/null +++ b/BeyondTools.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.VFS", "BeyondTools.VFS\BeyondTools.VFS.csproj", "{673078E8-FBE7-48B3-A228-E7451474625E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.SparkBuffer", "BeyondTools.SparkBuffer\BeyondTools.SparkBuffer.csproj", "{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.Build.0 = Release|Any CPU + {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md index c6e5aca..b43b568 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ -# BeyondTools - +## BeyondTools.VFS +Dumps VFS assets +``` +Usage: [arguments...] [-h|--help] [--version] +Arguments: + [0] game StreamingAssets path (Beyond_Data/StreamingAssets) + [1] dumped asset type + [2] output directory +``` +## BeyondTools.SparkBuffer +Dumps SparkBuffer binaries exported from VFS TextAsset +``` +Usage: [arguments...] [-h|--help] [--version] +Arguments: + [0] TextAsset/TableCfg directory + [1] output directory +``` \ No newline at end of file