diff --git a/BeyondTools.SparkBuffer/BeanType.cs b/BeyondTools.SparkBuffer/BeanType.cs
new file mode 100644
index 0000000..c982564
--- /dev/null
+++ b/BeyondTools.SparkBuffer/BeanType.cs
@@ -0,0 +1,95 @@
+using BeyondTools.SparkBuffer.Extensions;
+
+namespace BeyondTools.SparkBuffer
+{
+ public struct BeanType
+ {
+ public int typeHash;
+ public string name;
+ public Field[] fields;
+
+ public BeanType(BinaryReader reader)
+ {
+ typeHash = reader.ReadInt32();
+ name = reader.ReadSparkBufferString();
+ reader.Align4Bytes();
+ var fieldCount = reader.ReadInt32();
+ fields = new Field[fieldCount];
+
+ foreach (ref var field in fields.AsSpan())
+ {
+ field.name = reader.ReadSparkBufferString();
+ field.type = reader.ReadSparkType();
+ switch (field.type)
+ {
+ case SparkType.Bool:
+ case SparkType.Byte:
+ case SparkType.Int:
+ case SparkType.Long:
+ case SparkType.Float:
+ case SparkType.Double:
+ case SparkType.String:
+ break;
+ case SparkType.Enum:
+ case SparkType.Bean:
+ reader.Align4Bytes();
+ field.typeHash = reader.ReadInt32();
+ break;
+ case SparkType.Array:
+ field.type2 = reader.ReadSparkType();
+
+ if (field.type2.Value.IsEnumOrBeanType())
+ {
+ reader.Align4Bytes();
+ field.typeHash = reader.ReadInt32();
+ }
+ break;
+ case SparkType.Map:
+ field.type2 = reader.ReadSparkType();
+ field.type3 = reader.ReadSparkType();
+
+ if (field.type2.Value.IsEnumOrBeanType())
+ {
+ reader.Align4Bytes();
+ field.typeHash = reader.ReadInt32();
+ }
+ if (field.type3.Value.IsEnumOrBeanType())
+ {
+ reader.Align4Bytes();
+ field.typeHash2 = reader.ReadInt32();
+ }
+
+ break;
+ default:
+ throw new Exception(string.Format("Unsupported bean field type {0} at position {1}", field.type, reader.BaseStream.Position));
+ }
+ }
+ }
+
+ public struct Field
+ {
+ public string name;
+ public SparkType type;
+
+ ///
+ /// or key type
+ ///
+ public SparkType? type2;
+
+ ///
+ /// value type
+ ///
+ public SparkType? type3;
+
+ ///
+ /// , , , or key type hash
+ ///
+ public int? typeHash;
+
+ ///
+ /// value type hash
+ ///
+ public int? typeHash2;
+ }
+ }
+}
diff --git a/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj b/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj
new file mode 100644
index 0000000..88f11e2
--- /dev/null
+++ b/BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj
@@ -0,0 +1,16 @@
+
+
+
+ Exe
+ net9.0
+ enable
+ enable
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+
diff --git a/BeyondTools.SparkBuffer/EnumType.cs b/BeyondTools.SparkBuffer/EnumType.cs
new file mode 100644
index 0000000..5b35559
--- /dev/null
+++ b/BeyondTools.SparkBuffer/EnumType.cs
@@ -0,0 +1,33 @@
+using BeyondTools.SparkBuffer.Extensions;
+
+namespace BeyondTools.SparkBuffer
+{
+ public struct EnumType
+ {
+ public int typeHash;
+ public string name;
+ public EnumItem[] enums;
+
+ public EnumType(BinaryReader reader)
+ {
+ typeHash = reader.ReadInt32();
+ name = reader.ReadSparkBufferString();
+ reader.Align4Bytes();
+ var enumCount = reader.ReadInt32();
+ enums = new EnumItem[enumCount];
+
+ foreach (ref var enumItem in enums.AsSpan())
+ {
+ enumItem.name = reader.ReadSparkBufferString();
+ reader.Align4Bytes();
+ enumItem.value = reader.ReadInt32();
+ }
+ }
+
+ public struct EnumItem
+ {
+ public string name;
+ public int value;
+ }
+ }
+}
diff --git a/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs b/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs
new file mode 100644
index 0000000..4d460af
--- /dev/null
+++ b/BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs
@@ -0,0 +1,46 @@
+using System.IO;
+using System.Text;
+
+namespace BeyondTools.SparkBuffer.Extensions
+{
+ public static class BinaryReaderExtensions
+ {
+ public static SparkType ReadSparkType(this BinaryReader reader)
+ => (SparkType)reader.ReadByte();
+
+ public static long Seek(this BinaryReader reader, long pos, SeekOrigin seekOrigin = SeekOrigin.Begin)
+ => reader.BaseStream.Seek(pos, seekOrigin);
+
+ public static string ReadSparkBufferString(this BinaryReader reader)
+ {
+ using MemoryStream buffer = new();
+ while (true)
+ {
+ byte b = reader.ReadByte();
+ if (b == 0)
+ break;
+ buffer.WriteByte(b);
+ }
+
+ return Encoding.UTF8.GetString(buffer.ToArray());
+ }
+
+ public static string ReadSparkBufferStringOffset(this BinaryReader reader)
+ {
+ var stringOffset = reader.ReadInt32();
+ if (stringOffset == -1)
+ return string.Empty;
+
+ var oldPosition = reader.BaseStream.Position;
+
+ reader.Seek(stringOffset);
+ var tmp = reader.ReadSparkBufferString();
+
+ reader.BaseStream.Position = oldPosition;
+ return tmp;
+ }
+
+ public static void Align4Bytes(this BinaryReader reader)
+ => reader.Seek((reader.BaseStream.Position - 1) + (4 - ((reader.BaseStream.Position - 1) % 4)));
+ }
+}
diff --git a/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs b/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs
new file mode 100644
index 0000000..e0fe3ad
--- /dev/null
+++ b/BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs
@@ -0,0 +1,8 @@
+namespace BeyondTools.SparkBuffer.Extensions
+{
+ public static class SparkTypeExtensions
+ {
+ public static bool IsEnumOrBeanType(this SparkType type)
+ => type is SparkType.Enum or SparkType.Bean;
+ }
+}
diff --git a/BeyondTools.SparkBuffer/Properties/launchSettings.json b/BeyondTools.SparkBuffer/Properties/launchSettings.json
new file mode 100644
index 0000000..fb10990
--- /dev/null
+++ b/BeyondTools.SparkBuffer/Properties/launchSettings.json
@@ -0,0 +1,8 @@
+{
+ "profiles": {
+ "BeyondTools.SparkBuffer": {
+ "commandName": "Project",
+ "commandLineArgs": "C:\\EndfieldOut\\Assets\\TextAsset\\TableCfg C:\\EndfieldOut\\TableCfgOutput"
+ }
+ }
+}
\ No newline at end of file
diff --git a/BeyondTools.SparkBuffer/SparkBufferDump.cs b/BeyondTools.SparkBuffer/SparkBufferDump.cs
new file mode 100644
index 0000000..fb92251
--- /dev/null
+++ b/BeyondTools.SparkBuffer/SparkBufferDump.cs
@@ -0,0 +1,243 @@
+using BeyondTools.SparkBuffer.Extensions;
+using ConsoleAppFramework;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace BeyondTools.SparkBuffer
+{
+ internal class SparkBufferDump
+ {
+ static void Main(string[] args)
+ {
+ ConsoleApp.Run(args, (
+ [Argument] string tableCfgDir,
+ [Argument] string outputDir) =>
+ {
+ if (!Directory.Exists(tableCfgDir))
+ throw new FileNotFoundException($"{tableCfgDir} isn't a valid directory");
+ if (!Directory.Exists(outputDir))
+ Directory.CreateDirectory(outputDir);
+
+ var success = 0;
+ var processingCount = 0;
+ foreach (var tablePath in Directory.EnumerateFiles(tableCfgDir))
+ {
+ var fileName = Path.GetFileName(tablePath);
+
+ Console.WriteLine("Reading {0}...", fileName);
+ processingCount++;
+
+ using var file = File.OpenRead(tablePath);
+ using var binaryReader = new BinaryReader(file);
+
+ try
+ {
+ var typeDefOffset = binaryReader.ReadInt32();
+ var rootDefOffset = binaryReader.ReadInt32();
+ var dataOffset = binaryReader.ReadInt32();
+
+ binaryReader.Seek(typeDefOffset);
+ SparkManager.ReadTypeDefinitions(binaryReader);
+
+ binaryReader.Seek(rootDefOffset);
+ var rootDef = new BeanType.Field
+ {
+ type = binaryReader.ReadSparkType(),
+ name = binaryReader.ReadSparkBufferString()
+ };
+
+ if (rootDef.type.IsEnumOrBeanType())
+ {
+ binaryReader.Align4Bytes();
+ rootDef.typeHash = binaryReader.ReadInt32();
+ }
+ if (rootDef.type == SparkType.Map)
+ {
+ rootDef.type2 = binaryReader.ReadSparkType();
+ rootDef.type3 = binaryReader.ReadSparkType();
+
+ if (rootDef.type2.Value.IsEnumOrBeanType())
+ {
+ binaryReader.Align4Bytes();
+ rootDef.typeHash = binaryReader.ReadInt32();
+ }
+ if (rootDef.type3.Value.IsEnumOrBeanType())
+ {
+ binaryReader.Align4Bytes();
+ rootDef.typeHash2 = binaryReader.ReadInt32();
+ }
+ }
+
+ binaryReader.Seek(dataOffset);
+ var resultFilePath = Path.Combine(outputDir, $"{rootDef.name}.json");
+ switch (rootDef.type)
+ {
+ case SparkType.Bean:
+ var rootBeanType = SparkManager.BeanTypeFromHash((int)rootDef.typeHash!);
+ var beanDump = ReadBeanAsJObject(binaryReader, rootBeanType);
+ File.WriteAllText(resultFilePath, beanDump!.ToString());
+ break;
+ case SparkType.Map:
+ var mapDump = ReadMapAsJObject(binaryReader, rootDef);
+ File.WriteAllText(resultFilePath, JsonSerializer.Serialize(mapDump, SparkManager.jsonSerializerOptions));
+ break;
+ default:
+ throw new NotSupportedException(string.Format("Unsupported root type {0}", rootDef.type));
+ }
+
+ Console.WriteLine("Dumped {0} successfully", rootDef.name);
+ success++;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("Error in reading {0}, Error: {1}", fileName, ex.ToString());
+ }
+ }
+
+ Console.WriteLine("Dumped {0}/{1}", success, processingCount);
+ });
+ }
+
+ static JsonObject? ReadMapAsJObject(BinaryReader binaryReader, BeanType.Field typeDef)
+ {
+ var mapDump = new JsonObject();
+ var kvCount = binaryReader.ReadInt32();
+
+ for (int i = 0; i < kvCount; i++)
+ {
+ var key = typeDef.type2 switch
+ {
+ SparkType.String => binaryReader.ReadSparkBufferStringOffset(),
+ SparkType.Int => binaryReader.ReadInt32().ToString(),
+ _ => throw new NotSupportedException(string.Format("Unsupported map key type {0}", typeDef.type2)),
+ };
+ mapDump[key] = null;
+ }
+
+ for (int i = 0; i < kvCount; i++)
+ {
+ mapDump[i] = typeDef.type3 switch
+ {
+ SparkType.Bean => ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)typeDef.typeHash2!), true),
+ SparkType.String => binaryReader.ReadSparkBufferStringOffset(),
+ SparkType.Int => binaryReader.ReadInt32(),
+ _ => throw new NotSupportedException(string.Format("Unsupported map value type {0}", typeDef.type3)),
+ };
+ }
+
+ return mapDump;
+ }
+
+ static JsonObject? ReadBeanAsJObject(BinaryReader binaryReader, BeanType beanType, bool pointer = false)
+ {
+ long? pointerOrigin = null;
+ if (pointer)
+ {
+ var beanOffset = binaryReader.ReadInt32();
+ if (beanOffset == -1)
+ return null;
+
+ pointerOrigin = binaryReader.BaseStream.Position;
+ binaryReader.Seek(beanOffset);
+ }
+
+ var dumpObj = new JsonObject();
+
+ foreach (var (fieldIndex, beanField) in beanType.fields.Index())
+ {
+ long? origin = null;
+ if (beanField.type == SparkType.Array)
+ {
+ var fieldOffset = binaryReader.ReadInt32();
+ if (fieldOffset == -1)
+ {
+ dumpObj[beanField.name] = null;
+ continue;
+ }
+
+ origin = binaryReader.BaseStream.Position;
+ binaryReader.Seek(fieldOffset);
+ }
+
+ switch (beanField.type)
+ {
+ case SparkType.Array:
+ var jArray = new JsonArray();
+
+ var itemCount = binaryReader.ReadInt32();
+ while (itemCount-- > 0)
+ {
+ switch (beanField.type2)
+ {
+ case SparkType.String:
+ jArray.Add(binaryReader.ReadSparkBufferStringOffset());
+ break;
+ case SparkType.Bean:
+ jArray.Add(ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true));
+ break;
+ case SparkType.Float:
+ jArray.Add(binaryReader.ReadSingle());
+ break;
+ case SparkType.Long:
+ jArray.Add(binaryReader.ReadInt64());
+ break;
+ case SparkType.Int:
+ case SparkType.Enum:
+ jArray.Add(binaryReader.ReadInt32());
+ break;
+ case SparkType.Bool:
+ jArray.Add(binaryReader.ReadBoolean());
+ break;
+ default:
+ throw new NotSupportedException(string.Format("Unsupported array type {0} on bean field, position: {1}", beanField.type2, binaryReader.BaseStream.Position));
+ }
+ }
+
+ dumpObj[beanField.name] = jArray;
+ break;
+ case SparkType.Int:
+ case SparkType.Enum:
+ dumpObj[beanField.name] = binaryReader.ReadInt32();
+ break;
+ case SparkType.Long:
+ dumpObj[beanField.name] = binaryReader.ReadInt64();
+ break;
+ case SparkType.Float:
+ dumpObj[beanField.name] = binaryReader.ReadSingle();
+ break;
+ case SparkType.Double:
+ dumpObj[beanField.name] = binaryReader.ReadDouble();
+ break;
+ case SparkType.String:
+ dumpObj[beanField.name] = binaryReader.ReadSparkBufferStringOffset();
+ break;
+ case SparkType.Bean:
+ dumpObj[beanField.name] = ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true);
+ break;
+ case SparkType.Bool:
+ dumpObj[beanField.name] = binaryReader.ReadBoolean();
+ if (beanType.fields.Length > fieldIndex + 1 && beanType.fields[fieldIndex + 1].type != SparkType.Bool)
+ binaryReader.Align4Bytes();
+ break;
+ case SparkType.Map:
+ var mapOffset = binaryReader.ReadInt32();
+ var mapOrigin = binaryReader.BaseStream.Position;
+ binaryReader.Seek(mapOffset);
+ dumpObj[beanField.name] = ReadMapAsJObject(binaryReader, beanField);
+ binaryReader.Seek(mapOrigin);
+ break;
+ case SparkType.Byte:
+ throw new Exception(string.Format("Dumping bean field type {0} isn't supported, position: {1}", beanField.type, binaryReader.BaseStream.Position));
+ }
+
+ if (origin is not null)
+ binaryReader.BaseStream.Position = (long)origin;
+ }
+
+ if (pointerOrigin is not null)
+ binaryReader.BaseStream.Position = (long)pointerOrigin;
+
+ return dumpObj;
+ }
+ }
+}
diff --git a/BeyondTools.SparkBuffer/SparkManager.cs b/BeyondTools.SparkBuffer/SparkManager.cs
new file mode 100644
index 0000000..91d46ef
--- /dev/null
+++ b/BeyondTools.SparkBuffer/SparkManager.cs
@@ -0,0 +1,38 @@
+using BeyondTools.SparkBuffer.Extensions;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace BeyondTools.SparkBuffer
+{
+ public static class SparkManager
+ {
+ public static readonly JsonSerializerOptions jsonSerializerOptions = new() { IncludeFields = true, WriteIndented = true, NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals };
+
+ private static readonly Dictionary beanTypeMap = [];
+ private static readonly Dictionary enumTypeMap = [];
+
+ public static BeanType BeanTypeFromHash(int hash)
+ => beanTypeMap[hash];
+
+ public static void ReadTypeDefinitions(BinaryReader reader)
+ {
+ var typeDefCount = reader.ReadInt32();
+ while (typeDefCount-- > 0)
+ {
+ var sparkType = reader.ReadSparkType();
+ reader.Align4Bytes();
+
+ if (sparkType == SparkType.Enum)
+ {
+ var enumType = new EnumType(reader);
+ enumTypeMap.TryAdd(enumType.typeHash, enumType);
+ }
+ else if (sparkType == SparkType.Bean)
+ {
+ var beanType = new BeanType(reader);
+ beanTypeMap.TryAdd(beanType.typeHash, beanType);
+ }
+ }
+ }
+ }
+}
diff --git a/BeyondTools.SparkBuffer/SparkType.cs b/BeyondTools.SparkBuffer/SparkType.cs
new file mode 100644
index 0000000..56fcd1c
--- /dev/null
+++ b/BeyondTools.SparkBuffer/SparkType.cs
@@ -0,0 +1,17 @@
+namespace BeyondTools.SparkBuffer
+{
+ public enum SparkType : byte
+ {
+ Bool,
+ Byte,
+ Int,
+ Long,
+ Float,
+ Double,
+ Enum,
+ String,
+ Bean,
+ Array,
+ Map
+ }
+}
diff --git a/BeyondTools.VFS/BeyondTools.VFS.csproj b/BeyondTools.VFS/BeyondTools.VFS.csproj
new file mode 100644
index 0000000..e60ba34
--- /dev/null
+++ b/BeyondTools.VFS/BeyondTools.VFS.csproj
@@ -0,0 +1,18 @@
+
+
+
+ Exe
+ net9.0
+ enable
+ enable
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+
+
+
diff --git a/BeyondTools.VFS/Crypto/CSChaCha20.cs b/BeyondTools.VFS/Crypto/CSChaCha20.cs
new file mode 100644
index 0000000..6415cd2
--- /dev/null
+++ b/BeyondTools.VFS/Crypto/CSChaCha20.cs
@@ -0,0 +1,930 @@
+/*
+ * Copyright (c) 2015, 2018 Scott Bennett
+ * (c) 2018-2023 Kaarlo Räihä
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using System.Runtime.Intrinsics;
+using System.Runtime.CompilerServices;
+
+namespace BeyondTools.VFS.Crypto;
+
+///
+/// Chosen SIMD mode
+///
+public enum SimdMode
+{
+ ///
+ /// Autodetect
+ ///
+ AutoDetect = 0,
+
+ ///
+ /// 128 bit SIMD
+ ///
+ V128,
+
+ ///
+ /// 256 bit SIMD
+ ///
+ V256,
+
+ ///
+ /// 512 bit SIMD
+ ///
+ V512,
+
+ ///
+ /// No SIMD
+ ///
+ None
+}
+
+///
+/// Class for ChaCha20 encryption / decryption
+///
+public sealed class CSChaCha20 : IDisposable
+{
+ ///
+ /// Only allowed key lenght in bytes
+ ///
+ public const int allowedKeyLength = 32;
+
+ ///
+ /// Only allowed nonce lenght in bytes
+ ///
+ public const int allowedNonceLength = 12;
+
+ ///
+ /// How many bytes are processed per loop
+ ///
+ public const int processBytesAtTime = 64;
+
+ private const int stateLength = 16;
+
+ ///
+ /// The ChaCha20 state (aka "context")
+ ///
+ private readonly uint[] state = new uint[stateLength];
+
+ ///
+ /// Determines if the objects in this class have been disposed of. Set to true by the Dispose() method.
+ ///
+ private bool isDisposed = false;
+
+ ///
+ /// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens.
+ ///
+ ///
+ /// See ChaCha20 Spec Section 2.4 for a detailed description of the inputs.
+ ///
+ ///
+ /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers
+ ///
+ ///
+ /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers
+ ///
+ ///
+ /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer
+ ///
+ public CSChaCha20(byte[] key, byte[] nonce, uint counter)
+ {
+ KeySetup(key);
+ IvSetup(nonce, counter);
+ }
+
+ ///
+ /// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens.
+ ///
+ ///
+ /// See ChaCha20 Spec Section 2.4 for a detailed description of the inputs.
+ ///
+ /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers
+ /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers
+ /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian unsigned integer
+ public CSChaCha20(ReadOnlySpan key, ReadOnlySpan nonce, uint counter)
+ {
+ KeySetup(key.ToArray());
+ IvSetup(nonce.ToArray(), counter);
+ }
+
+ ///
+ /// The ChaCha20 state (aka "context"). Read-Only.
+ ///
+ public uint[] State
+ {
+ get
+ {
+ return state;
+ }
+ }
+
+
+ // These are the same constants defined in the reference implementation.
+ // http://cr.yp.to/streamciphers/timings/estreambench/submissions/salsa20/chacha8/ref/chacha.c
+ private static readonly byte[] sigma = "expand 32-byte k"u8.ToArray();
+ private static readonly byte[] tau = "expand 16-byte k"u8.ToArray();
+
+ ///
+ /// Set up the ChaCha state with the given key. A 32-byte key is required and enforced.
+ ///
+ ///
+ /// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers
+ ///
+ private void KeySetup(byte[] key)
+ {
+ if (key == null)
+ {
+ throw new ArgumentNullException("Key is null");
+ }
+
+ if (key.Length != allowedKeyLength)
+ {
+ throw new ArgumentException($"Key length must be {allowedKeyLength}. Actual: {key.Length}");
+ }
+
+ state[4] = Util.U8To32Little(key, 0);
+ state[5] = Util.U8To32Little(key, 4);
+ state[6] = Util.U8To32Little(key, 8);
+ state[7] = Util.U8To32Little(key, 12);
+
+ byte[] constants = key.Length == allowedKeyLength ? sigma : tau;
+ int keyIndex = key.Length - 16;
+
+ state[8] = Util.U8To32Little(key, keyIndex + 0);
+ state[9] = Util.U8To32Little(key, keyIndex + 4);
+ state[10] = Util.U8To32Little(key, keyIndex + 8);
+ state[11] = Util.U8To32Little(key, keyIndex + 12);
+
+ state[0] = Util.U8To32Little(constants, 0);
+ state[1] = Util.U8To32Little(constants, 4);
+ state[2] = Util.U8To32Little(constants, 8);
+ state[3] = Util.U8To32Little(constants, 12);
+ }
+
+ ///
+ /// Set up the ChaCha state with the given nonce (aka Initialization Vector or IV) and block counter. A 12-byte nonce and a 4-byte counter are required.
+ ///
+ ///
+ /// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers
+ ///
+ ///
+ /// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer
+ ///
+ private void IvSetup(byte[] nonce, uint counter)
+ {
+ if (nonce == null)
+ {
+ // There has already been some state set up. Clear it before exiting.
+ Dispose();
+ throw new ArgumentNullException("Nonce is null");
+ }
+
+ if (nonce.Length != allowedNonceLength)
+ {
+ // There has already been some state set up. Clear it before exiting.
+ Dispose();
+ throw new ArgumentException($"Nonce length must be {allowedNonceLength}. Actual: {nonce.Length}");
+ }
+
+ state[12] = counter;
+ state[13] = Util.U8To32Little(nonce, 0);
+ state[14] = Util.U8To32Little(nonce, 4);
+ state[15] = Util.U8To32Little(nonce, 8);
+ }
+
+ private static SimdMode DetectSimdMode()
+ {
+ if (Vector512.IsHardwareAccelerated)
+ {
+ return SimdMode.V512;
+ }
+ else if (Vector256.IsHardwareAccelerated)
+ {
+ return SimdMode.V256;
+ }
+ else if (Vector128.IsHardwareAccelerated)
+ {
+ return SimdMode.V128;
+ }
+
+ return SimdMode.None;
+ }
+
+ #region Encryption methods
+
+ ///
+ /// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Output byte array, must have enough bytes
+ /// Input byte array
+ /// Number of bytes to encrypt
+ /// Chosen SIMD mode (default is auto-detect)
+ public void EncryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (output == null)
+ {
+ throw new ArgumentNullException("output", "Output cannot be null");
+ }
+
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (numBytes < 0 || numBytes > input.Length)
+ {
+ throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
+ }
+
+ if (output.Length < numBytes)
+ {
+ throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkBytes(output, input, numBytes, simdMode);
+ }
+
+ ///
+ /// Encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
+ ///
+ /// Output stream
+ /// Input stream
+ /// How many bytes to read and write at time, default is 1024
+ /// Chosen SIMD mode (default is auto-detect)
+ public void EncryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime);
+ }
+
+ ///
+ /// Async encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
+ ///
+ /// Output stream
+ /// Input stream
+ /// How many bytes to read and write at time, default is 1024
+ /// Chosen SIMD mode (default is auto-detect)
+ public async Task EncryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime);
+ }
+
+ ///
+ /// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Output byte array, must have enough bytes
+ /// Input byte array
+ /// Chosen SIMD mode (default is auto-detect)
+ public void EncryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (output == null)
+ {
+ throw new ArgumentNullException("output", "Output cannot be null");
+ }
+
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkBytes(output, input, input.Length, simdMode);
+ }
+
+ ///
+ /// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Input byte array
+ /// Number of bytes to encrypt
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains encrypted bytes
+ public byte[] EncryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (numBytes < 0 || numBytes > input.Length)
+ {
+ throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] returnArray = new byte[numBytes];
+ WorkBytes(returnArray, input, numBytes, simdMode);
+ return returnArray;
+ }
+
+ ///
+ /// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Input byte array
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains encrypted bytes
+ public byte[] EncryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] returnArray = new byte[input.Length];
+ WorkBytes(returnArray, input, input.Length, simdMode);
+ return returnArray;
+ }
+
+ ///
+ /// Encrypt string as UTF8 byte array, returns byte array that is allocated by method.
+ ///
+ /// Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform
+ /// Input string
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains encrypted bytes
+ public byte[] EncryptString(string input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] utf8Bytes = System.Text.Encoding.UTF8.GetBytes(input);
+ byte[] returnArray = new byte[utf8Bytes.Length];
+
+ WorkBytes(returnArray, utf8Bytes, utf8Bytes.Length, simdMode);
+ return returnArray;
+ }
+
+ #endregion // Encryption methods
+
+
+ #region // Decryption methods
+
+ ///
+ /// Decrypt arbitrary-length byte array (input), writing the resulting byte array to the output buffer.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Output byte array
+ /// Input byte array
+ /// Number of bytes to decrypt
+ /// Chosen SIMD mode (default is auto-detect)
+ public void DecryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (output == null)
+ {
+ throw new ArgumentNullException("output", "Output cannot be null");
+ }
+
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (numBytes < 0 || numBytes > input.Length)
+ {
+ throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
+ }
+
+ if (output.Length < numBytes)
+ {
+ throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkBytes(output, input, numBytes, simdMode);
+ }
+
+ ///
+ /// Decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
+ ///
+ /// Output stream
+ /// Input stream
+ /// How many bytes to read and write at time, default is 1024
+ /// Chosen SIMD mode (default is auto-detect)
+ public void DecryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime);
+ }
+
+ ///
+ /// Async decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
+ ///
+ /// Output stream
+ /// Input stream
+ /// How many bytes to read and write at time, default is 1024
+ /// Chosen SIMD mode (default is auto-detect)
+ public async Task DecryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime);
+ }
+
+ ///
+ /// Decrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Output byte array, must have enough bytes
+ /// Input byte array
+ /// Chosen SIMD mode (default is auto-detect)
+ public void DecryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (output == null)
+ {
+ throw new ArgumentNullException("output", "Output cannot be null");
+ }
+
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ WorkBytes(output, input, input.Length, simdMode);
+ }
+
+ ///
+ /// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Input byte array
+ /// Number of bytes to encrypt
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains decrypted bytes
+ public byte[] DecryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (numBytes < 0 || numBytes > input.Length)
+ {
+ throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] returnArray = new byte[numBytes];
+ WorkBytes(returnArray, input, numBytes, simdMode);
+ return returnArray;
+ }
+
+ ///
+ /// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
+ ///
+ /// Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method
+ /// Input byte array
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains decrypted bytes
+ public byte[] DecryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] returnArray = new byte[input.Length];
+ WorkBytes(returnArray, input, input.Length, simdMode);
+ return returnArray;
+ }
+
+ ///
+ /// Decrypt UTF8 byte array to string.
+ ///
+ /// Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform
+ /// Byte array
+ /// Chosen SIMD mode (default is auto-detect)
+ /// Byte array that contains encrypted bytes
+ public string DecryptUTF8ByteArray(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
+ {
+ if (input == null)
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
+
+ if (simdMode == SimdMode.AutoDetect)
+ {
+ simdMode = DetectSimdMode();
+ }
+
+ byte[] tempArray = new byte[input.Length];
+
+ WorkBytes(tempArray, input, input.Length, simdMode);
+ return System.Text.Encoding.UTF8.GetString(tempArray);
+ }
+
+ #endregion // Decryption methods
+
+ private void WorkStreams(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024)
+ {
+ int readBytes;
+
+ byte[] inputBuffer = new byte[howManyBytesToProcessAtTime];
+ byte[] outputBuffer = new byte[howManyBytesToProcessAtTime];
+
+ while ((readBytes = input.Read(inputBuffer, 0, howManyBytesToProcessAtTime)) > 0)
+ {
+ // Encrypt or decrypt
+ WorkBytes(output: outputBuffer, input: inputBuffer, numBytes: readBytes, simdMode);
+
+ // Write buffer
+ output.Write(outputBuffer, 0, readBytes);
+ }
+ }
+
+ private async Task WorkStreamsAsync(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024)
+ {
+ byte[] readBytesBuffer = new byte[howManyBytesToProcessAtTime];
+ byte[] writeBytesBuffer = new byte[howManyBytesToProcessAtTime];
+ int howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime);
+
+ while (howManyBytesWereRead > 0)
+ {
+ // Encrypt or decrypt
+ WorkBytes(output: writeBytesBuffer, input: readBytesBuffer, numBytes: howManyBytesWereRead, simdMode);
+
+ // Write
+ await output.WriteAsync(writeBytesBuffer, 0, howManyBytesWereRead);
+
+ // Read more
+ howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime);
+ }
+ }
+
+ ///
+ /// Encrypt or decrypt an arbitrary-length byte array (input), writing the resulting byte array to the output buffer. The number of bytes to read from the input buffer is determined by numBytes.
+ ///
+ /// Output byte array
+ /// Input byte array
+ /// How many bytes to process
+ /// Chosen SIMD mode (default is auto-detect)
+ private void WorkBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode)
+ {
+ if (isDisposed)
+ {
+ throw new ObjectDisposedException("state", "The ChaCha state has been disposed");
+ }
+
+ uint[] x = new uint[stateLength]; // Working buffer
+ byte[] tmp = new byte[processBytesAtTime]; // Temporary buffer
+ int offset = 0;
+
+ int howManyFullLoops = numBytes / processBytesAtTime;
+ int tailByteCount = numBytes - howManyFullLoops * processBytesAtTime;
+
+ for (int loop = 0; loop < howManyFullLoops; loop++)
+ {
+ UpdateStateAndGenerateTemporaryBuffer(state, x, tmp);
+
+ if (simdMode == SimdMode.V512)
+ {
+ // 1 x 64 bytes
+ Vector512 inputV = Vector512.Create(input, offset);
+ Vector512 tmpV = Vector512.Create(tmp, 0);
+ Vector512 outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset);
+ }
+ else if (simdMode == SimdMode.V256)
+ {
+ // 2 x 32 bytes
+ Vector256 inputV = Vector256.Create(input, offset);
+ Vector256 tmpV = Vector256.Create(tmp, 0);
+ Vector256 outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset);
+
+ inputV = Vector256.Create(input, offset + 32);
+ tmpV = Vector256.Create(tmp, 32);
+ outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset + 32);
+ }
+ else if (simdMode == SimdMode.V128)
+ {
+ // 4 x 16 bytes
+ Vector128 inputV = Vector128.Create(input, offset);
+ Vector128 tmpV = Vector128.Create(tmp, 0);
+ Vector128 outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset);
+
+ inputV = Vector128.Create(input, offset + 16);
+ tmpV = Vector128.Create(tmp, 16);
+ outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset + 16);
+
+ inputV = Vector128.Create(input, offset + 32);
+ tmpV = Vector128.Create(tmp, 32);
+ outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset + 32);
+
+ inputV = Vector128.Create(input, offset + 48);
+ tmpV = Vector128.Create(tmp, 48);
+ outputV = inputV ^ tmpV;
+ outputV.CopyTo(output, offset + 48);
+ }
+ else
+ {
+ for (int i = 0; i < processBytesAtTime; i += 4)
+ {
+ // Small unroll
+ int start = i + offset;
+ output[start] = (byte)(input[start] ^ tmp[i]);
+ output[start + 1] = (byte)(input[start + 1] ^ tmp[i + 1]);
+ output[start + 2] = (byte)(input[start + 2] ^ tmp[i + 2]);
+ output[start + 3] = (byte)(input[start + 3] ^ tmp[i + 3]);
+ }
+ }
+
+ offset += processBytesAtTime;
+ }
+
+ // In case there are some bytes left
+ if (tailByteCount > 0)
+ {
+ UpdateStateAndGenerateTemporaryBuffer(state, x, tmp);
+
+ for (int i = 0; i < tailByteCount; i++)
+ {
+ output[i + offset] = (byte)(input[i + offset] ^ tmp[i]);
+ }
+ }
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static void UpdateStateAndGenerateTemporaryBuffer(uint[] stateToModify, uint[] workingBuffer, byte[] temporaryBuffer)
+ {
+ // Copy state to working buffer
+ Buffer.BlockCopy(stateToModify, 0, workingBuffer, 0, stateLength * sizeof(uint));
+
+ for (int i = 0; i < 10; i++)
+ {
+ QuarterRound(workingBuffer, 0, 4, 8, 12);
+ QuarterRound(workingBuffer, 1, 5, 9, 13);
+ QuarterRound(workingBuffer, 2, 6, 10, 14);
+ QuarterRound(workingBuffer, 3, 7, 11, 15);
+
+ QuarterRound(workingBuffer, 0, 5, 10, 15);
+ QuarterRound(workingBuffer, 1, 6, 11, 12);
+ QuarterRound(workingBuffer, 2, 7, 8, 13);
+ QuarterRound(workingBuffer, 3, 4, 9, 14);
+ }
+
+ for (int i = 0; i < stateLength; i++)
+ {
+ Util.ToBytes(temporaryBuffer, Util.Add(workingBuffer[i], stateToModify[i]), 4 * i);
+ }
+
+ stateToModify[12] = Util.AddOne(stateToModify[12]);
+ if (stateToModify[12] <= 0)
+ {
+ /* Stopping at 2^70 bytes per nonce is the user's responsibility */
+ stateToModify[13] = Util.AddOne(stateToModify[13]);
+ }
+ }
+
+ ///
+ /// The ChaCha Quarter Round operation. It operates on four 32-bit unsigned integers within the given buffer at indices a, b, c, and d.
+ ///
+ ///
+ /// The ChaCha state does not have four integer numbers: it has 16. So the quarter-round operation works on only four of them -- hence the name. Each quarter round operates on four predetermined numbers in the ChaCha state.
+ /// See ChaCha20 Spec Sections 2.1 - 2.2.
+ ///
+ /// A ChaCha state (vector). Must contain 16 elements.
+ /// Index of the first number
+ /// Index of the second number
+ /// Index of the third number
+ /// Index of the fourth number
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static void QuarterRound(uint[] x, uint a, uint b, uint c, uint d)
+ {
+ x[a] = Util.Add(x[a], x[b]);
+ x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 16);
+
+ x[c] = Util.Add(x[c], x[d]);
+ x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 12);
+
+ x[a] = Util.Add(x[a], x[b]);
+ x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 8);
+
+ x[c] = Util.Add(x[c], x[d]);
+ x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 7);
+ }
+
+ #region Destructor and Disposer
+
+ ///
+ /// Clear and dispose of the internal state. The finalizer is only called if Dispose() was never called on this cipher.
+ ///
+ ~CSChaCha20()
+ {
+ Dispose(false);
+ }
+
+ ///
+ /// Clear and dispose of the internal state. Also request the GC not to call the finalizer, because all cleanup has been taken care of.
+ ///
+ public void Dispose()
+ {
+ Dispose(true);
+ /*
+ * The Garbage Collector does not need to invoke the finalizer because Dispose(bool) has already done all the cleanup needed.
+ */
+ GC.SuppressFinalize(this);
+ }
+
+ ///
+ /// This method should only be invoked from Dispose() or the finalizer. This handles the actual cleanup of the resources.
+ ///
+ ///
+ /// Should be true if called by Dispose(); false if called by the finalizer
+ ///
+ private void Dispose(bool disposing)
+ {
+ if (!isDisposed)
+ {
+ if (disposing)
+ {
+ /* Cleanup managed objects by calling their Dispose() methods */
+ }
+
+ /* Cleanup any unmanaged objects here */
+ Array.Clear(state, 0, stateLength);
+ }
+
+ isDisposed = true;
+ }
+
+ #endregion // Destructor and Disposer
+}
+
+///
+/// Utilities that are used during compression
+///
+public static class Util
+{
+ ///
+ /// n-bit left rotation operation (towards the high bits) for 32-bit integers.
+ ///
+ ///
+ ///
+ /// The result of (v LEFTSHIFT c)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static uint Rotate(uint v, int c)
+ {
+ unchecked
+ {
+ return v << c | v >> 32 - c;
+ }
+ }
+
+ ///
+ /// Unchecked integer exclusive or (XOR) operation.
+ ///
+ ///
+ ///
+ /// The result of (v XOR w)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static uint XOr(uint v, uint w)
+ {
+ return unchecked(v ^ w);
+ }
+
+ ///
+ /// Unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32.
+ ///
+ ///
+ /// See ChaCha20 Spec Section 2.1.
+ ///
+ ///
+ ///
+ /// The result of (v + w) modulo 2^32
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static uint Add(uint v, uint w)
+ {
+ return unchecked(v + w);
+ }
+
+ ///
+ /// Add 1 to the input parameter using unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32.
+ ///
+ ///
+ /// See ChaCha20 Spec Section 2.1.
+ ///
+ ///
+ /// The result of (v + 1) modulo 2^32
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static uint AddOne(uint v)
+ {
+ return unchecked(v + 1);
+ }
+
+ ///
+ /// Convert four bytes of the input buffer into an unsigned 32-bit integer, beginning at the inputOffset.
+ ///
+ ///
+ ///
+ /// An unsigned 32-bit integer
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static uint U8To32Little(byte[] p, int inputOffset)
+ {
+ unchecked
+ {
+ return p[inputOffset]
+ | (uint)p[inputOffset + 1] << 8
+ | (uint)p[inputOffset + 2] << 16
+ | (uint)p[inputOffset + 3] << 24;
+ }
+ }
+
+ ///
+ /// Serialize the input integer into the output buffer. The input integer will be split into 4 bytes and put into four sequential places in the output buffer, starting at the outputOffset.
+ ///
+ ///
+ ///
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void ToBytes(byte[] output, uint input, int outputOffset)
+ {
+ unchecked
+ {
+ output[outputOffset] = (byte)input;
+ output[outputOffset + 1] = (byte)(input >> 8);
+ output[outputOffset + 2] = (byte)(input >> 16);
+ output[outputOffset + 3] = (byte)(input >> 24);
+ }
+ }
+}
\ No newline at end of file
diff --git a/BeyondTools.VFS/EVFSBlockType.cs b/BeyondTools.VFS/EVFSBlockType.cs
new file mode 100644
index 0000000..d2f5db1
--- /dev/null
+++ b/BeyondTools.VFS/EVFSBlockType.cs
@@ -0,0 +1,20 @@
+namespace BeyondTools.VFS
+{
+ public enum EVFSBlockType : byte
+ {
+ All,
+
+ InitialAudio = 1,
+ InitialBundle,
+ BundleManifest,
+ LowShader,
+ Audio = 11,
+ Bundle,
+ TextAsset = 14,
+ Video,
+ IV,
+ Streaming,
+ IFixPatch = 21,
+ Raw = 31
+ }
+}
diff --git a/BeyondTools.VFS/EVFSLoaderPosType.cs b/BeyondTools.VFS/EVFSLoaderPosType.cs
new file mode 100644
index 0000000..cbd70bb
--- /dev/null
+++ b/BeyondTools.VFS/EVFSLoaderPosType.cs
@@ -0,0 +1,13 @@
+namespace BeyondTools.VFS
+{
+ public enum EVFSLoaderPosType : byte
+ {
+ None,
+ PersistAsset,
+ StreamAsset,
+ VFS = 10,
+ VFS_PersistAsset,
+ VFS_StreamAsset,
+ VFS_Build
+ }
+}
diff --git a/BeyondTools.VFS/Extensions/StreamExtensions.cs b/BeyondTools.VFS/Extensions/StreamExtensions.cs
new file mode 100644
index 0000000..dc321b5
--- /dev/null
+++ b/BeyondTools.VFS/Extensions/StreamExtensions.cs
@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BeyondTools.VFS.Extensions
+{
+ internal static class StreamExtensions
+ {
+ public static void CopyBytes(this Stream inStream, Stream outStream, long? count = null)
+ {
+ if (count == null)
+ {
+ inStream.CopyTo(outStream);
+ return;
+ }
+
+ long readBytes = 0L;
+ var buffer = new byte[64 * 1024];
+ do
+ {
+ var toRead = Math.Min((long)(count - readBytes), buffer.LongLength);
+ var readNow = inStream.Read(buffer, 0, (int)toRead);
+ if (readNow == 0)
+ break;
+ outStream.Write(buffer, 0, readNow);
+ readBytes += readNow;
+ } while (readBytes < count);
+ }
+ }
+}
diff --git a/BeyondTools.VFS/FVFBlockChunkInfo.cs b/BeyondTools.VFS/FVFBlockChunkInfo.cs
new file mode 100644
index 0000000..237fb4d
--- /dev/null
+++ b/BeyondTools.VFS/FVFBlockChunkInfo.cs
@@ -0,0 +1,16 @@
+using System.Text.Json.Serialization;
+
+namespace BeyondTools.VFS
+{
+ public struct FVFBlockChunkInfo
+ {
+ public const string FILE_EXTENSION = ".chk";
+
+ public UInt128 md5Name;
+ public UInt128 contentMD5;
+ public long length;
+ public EVFSBlockType blockType;
+ [JsonIgnore]
+ public FVFBlockFileInfo[] files;
+ }
+}
diff --git a/BeyondTools.VFS/FVFBlockFileInfo.cs b/BeyondTools.VFS/FVFBlockFileInfo.cs
new file mode 100644
index 0000000..a404321
--- /dev/null
+++ b/BeyondTools.VFS/FVFBlockFileInfo.cs
@@ -0,0 +1,17 @@
+namespace BeyondTools.VFS
+{
+ public struct FVFBlockFileInfo
+ {
+ public string fileName;
+ public long fileNameHash;
+ public UInt128 fileChunkMD5Name;
+ public UInt128 fileDataMD5;
+ public long offset;
+ public long len;
+ public EVFSBlockType blockType;
+ public bool bUseEncrypt;
+ public long ivSeed;
+ public bool bIsDirect;
+ public EVFSLoaderPosType loaderPosType;
+ }
+}
diff --git a/BeyondTools.VFS/Properties/launchSettings.json b/BeyondTools.VFS/Properties/launchSettings.json
new file mode 100644
index 0000000..5ca7bc3
--- /dev/null
+++ b/BeyondTools.VFS/Properties/launchSettings.json
@@ -0,0 +1,8 @@
+{
+ "profiles": {
+ "BeyondTools.VFS": {
+ "commandName": "Project",
+ "commandLineArgs": "C:/Users/rafi1/Desktop/etc/Beyond_TMP/Game/Beyond_Data/StreamingAssets InitialBundle C:/EndfieldOut/Assets"
+ }
+ }
+}
\ No newline at end of file
diff --git a/BeyondTools.VFS/VFBlockMainInfo.cs b/BeyondTools.VFS/VFBlockMainInfo.cs
new file mode 100644
index 0000000..8ea4225
--- /dev/null
+++ b/BeyondTools.VFS/VFBlockMainInfo.cs
@@ -0,0 +1,95 @@
+using System.Buffers.Binary;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace BeyondTools.VFS
+{
+ public class VFBlockMainInfo
+ {
+ public VFBlockMainInfo(byte[] bytes, int offset = 0)
+ {
+ version = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(int);
+ // TODO: CRC and stuff idk
+ offset += 12;
+
+ ushort groupCfgNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(ushort);
+ groupCfgName = Encoding.UTF8.GetString(bytes.AsSpan(offset, groupCfgNameLength));
+ offset += groupCfgNameLength;
+
+ groupCfgHashName = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ groupFileInfoNum = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(int);
+
+ groupChunksLength = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ blockType = (EVFSBlockType)bytes[offset++];
+
+ var chunkCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
+ allChunks = GC.AllocateUninitializedArray(chunkCount);
+ offset += sizeof(int);
+
+ foreach (ref var chunk in allChunks.AsSpan())
+ {
+ chunk.md5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
+ offset += Marshal.SizeOf();
+
+ chunk.contentMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
+ offset += Marshal.SizeOf();
+
+ chunk.length = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ chunk.blockType = (EVFSBlockType)bytes[offset++];
+
+ var fileCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
+ chunk.files = GC.AllocateUninitializedArray(fileCount);
+ offset += sizeof(int);
+
+ foreach (ref var file in chunk.files.AsSpan())
+ {
+ ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(ushort);
+ file.fileName = Encoding.UTF8.GetString(bytes.AsSpan(offset, fileNameLength));
+ offset += fileNameLength;
+
+ file.fileNameHash = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ file.fileChunkMD5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
+ offset += Marshal.SizeOf();
+
+ file.fileDataMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
+ offset += Marshal.SizeOf();
+
+ file.offset = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ file.len = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+
+ file.blockType = (EVFSBlockType)bytes[offset++];
+ file.bUseEncrypt = Convert.ToBoolean(bytes[offset++]);
+ if (file.bUseEncrypt)
+ {
+ file.ivSeed = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
+ offset += sizeof(long);
+ }
+ }
+ }
+
+ }
+
+ public int version;
+ public string groupCfgName;
+ public long groupCfgHashName;
+ public int groupFileInfoNum;
+ public long groupChunksLength;
+ public EVFSBlockType blockType;
+ public FVFBlockChunkInfo[] allChunks;
+ }
+}
diff --git a/BeyondTools.VFS/VFSDefine.cs b/BeyondTools.VFS/VFSDefine.cs
new file mode 100644
index 0000000..81f0314
--- /dev/null
+++ b/BeyondTools.VFS/VFSDefine.cs
@@ -0,0 +1,11 @@
+namespace BeyondTools.VFS
+{
+ public static class VFSDefine
+ {
+ public const string CHACHA_KEY = "eU1cu+MYQiaYdVherRzV86pv/N/lIU/9gIk+5n5Vj4Y=";
+ public const string VFS_DIR = "VFS";
+ public const int VFS_PROTO_VERSION = 3;
+ public const int VFS_VFB_HEAD_LEN = 16;
+ public const int BLOCK_HEAD_LEN = 12;
+ }
+}
diff --git a/BeyondTools.VFS/VFSDump.cs b/BeyondTools.VFS/VFSDump.cs
new file mode 100644
index 0000000..fe3d893
--- /dev/null
+++ b/BeyondTools.VFS/VFSDump.cs
@@ -0,0 +1,103 @@
+using BeyondTools.VFS.Crypto;
+using BeyondTools.VFS.Extensions;
+using ConsoleAppFramework;
+using System.IO.Hashing;
+using System.Text;
+
+namespace BeyondTools.VFS;
+internal class VFSDump
+{
+ static readonly Dictionary blockTypeMap = new()
+ {
+ { EVFSBlockType.Audio, "MainAudio" },
+ { EVFSBlockType.Bundle, "MainBundles" },
+ { EVFSBlockType.BundleManifest, "BundleManifest" },
+ { EVFSBlockType.IFixPatch, "IFixPatchOut" },
+ { EVFSBlockType.InitialAudio, "InitAudio" },
+ { EVFSBlockType.InitialBundle, "InitBundles" },
+ { EVFSBlockType.IV, "IV" },
+ { EVFSBlockType.LowShader, "LowShader" },
+ // { EVFSBlockType.Raw, "" }, Not present
+ { EVFSBlockType.Streaming, "Streaming" },
+ { EVFSBlockType.TextAsset, "TextAsset" },
+ { EVFSBlockType.Video, "Video" },
+ };
+
+ static void Main(string[] args)
+ {
+ ConsoleApp.Run(args, (
+ [Argument] string streamingAssetsPath,
+ [Argument] EVFSBlockType dumpAssetType = EVFSBlockType.All,
+ [Argument] string? outputDir = null) =>
+ {
+ streamingAssetsPath = Path.Combine(streamingAssetsPath, VFSDefine.VFS_DIR);
+ outputDir ??= Path.Combine(AppContext.BaseDirectory, "Assets");
+ if (dumpAssetType == EVFSBlockType.All)
+ {
+ foreach (var type in blockTypeMap.Keys)
+ {
+ DumpAssetByType(streamingAssetsPath, type, outputDir);
+ }
+ }
+ else
+ {
+ DumpAssetByType(streamingAssetsPath, dumpAssetType, outputDir);
+ }
+ });
+ }
+
+ private static void DumpAssetByType(string streamingAssetsPath, EVFSBlockType dumpAssetType, string outputDir)
+ {
+ Console.WriteLine("Dumping {0} files...", dumpAssetType.ToString());
+
+ // TODO: This is a temporary solution that makes this thing only worked on some EVFSBlockType, i haven't been able to figure out Crc32Utils.UnityCRC64
+ var blockDir = Directory.EnumerateDirectories(streamingAssetsPath).First(x => x.Split('/', '\\').Last().StartsWith(Convert.ToHexString(Crc32.Hash(Encoding.UTF8.GetBytes(blockTypeMap[dumpAssetType])))));
+ var blockFilePath = Path.Combine(blockDir, blockDir.Split('/', '\\').Last() + ".blc");
+
+ var blockFile = File.ReadAllBytes(blockFilePath);
+ byte[] nonce = GC.AllocateUninitializedArray(VFSDefine.BLOCK_HEAD_LEN);
+ Buffer.BlockCopy(blockFile, 0, nonce, 0, nonce.Length);
+
+ var chacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), nonce, 1);
+ var decryptedBytes = chacha.DecryptBytes(blockFile[VFSDefine.BLOCK_HEAD_LEN..]);
+ Buffer.BlockCopy(decryptedBytes, 0, blockFile, VFSDefine.BLOCK_HEAD_LEN, decryptedBytes.Length);
+
+ var vfBlockMainInfo = new VFBlockMainInfo(blockFile);
+ foreach (var chunk in vfBlockMainInfo.allChunks)
+ {
+ var chunkMd5Name = Convert.ToHexString(BitConverter.GetBytes(chunk.md5Name)) + FVFBlockChunkInfo.FILE_EXTENSION;
+ var chunkFs = File.OpenRead(Path.Join(blockDir, chunkMd5Name));
+ foreach (var file in chunk.files)
+ {
+ var filePath = Path.Combine(outputDir, file.fileName);
+ if (!Directory.Exists(Path.GetDirectoryName(filePath)))
+ Directory.CreateDirectory(Path.GetDirectoryName(filePath) ?? throw new InvalidDataException($"Cannot get directory name of {filePath}"));
+
+ if (file.bUseEncrypt)
+ {
+ byte[] fileNonce = GC.AllocateUninitializedArray(VFSDefine.BLOCK_HEAD_LEN);
+ Buffer.BlockCopy(BitConverter.GetBytes(vfBlockMainInfo.version), 0, fileNonce, 0, sizeof(int));
+ Buffer.BlockCopy(BitConverter.GetBytes(file.ivSeed), 0, fileNonce, sizeof(int), sizeof(long));
+
+ var fileChacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), fileNonce, 1);
+ var encryptedMs = new MemoryStream();
+ chunkFs.CopyBytes(encryptedMs, file.len);
+
+ // TODO: Should've used stream decryptor for better perf, but idk how to get it working
+ File.WriteAllBytes(filePath, fileChacha.DecryptBytes(encryptedMs.ToArray()));
+ }
+ else
+ {
+ var fileFs = File.OpenWrite(filePath);
+ chunkFs.Seek(file.offset, SeekOrigin.Begin);
+ chunkFs.CopyBytes(fileFs, file.len);
+ fileFs.Dispose();
+ }
+
+ }
+
+ Console.WriteLine("Dumped {0} file(s) from chunk {1}", chunk.files.Length, chunkMd5Name);
+ chunkFs.Dispose();
+ }
+ }
+}
\ No newline at end of file
diff --git a/BeyondTools.sln b/BeyondTools.sln
new file mode 100644
index 0000000..c9c0a34
--- /dev/null
+++ b/BeyondTools.sln
@@ -0,0 +1,28 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.12.35527.113
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.VFS", "BeyondTools.VFS\BeyondTools.VFS.csproj", "{673078E8-FBE7-48B3-A228-E7451474625E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.SparkBuffer", "BeyondTools.SparkBuffer\BeyondTools.SparkBuffer.csproj", "{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Release|Any CPU = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.Build.0 = Release|Any CPU
+ {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.Build.0 = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/README.md b/README.md
index c6e5aca..b43b568 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,17 @@
-# BeyondTools
-
+## BeyondTools.VFS
+Dumps VFS assets
+```
+Usage: [arguments...] [-h|--help] [--version]
+Arguments:
+ [0] game StreamingAssets path (Beyond_Data/StreamingAssets)
+ [1] dumped asset type
+ [2] output directory
+```
+## BeyondTools.SparkBuffer
+Dumps SparkBuffer binaries exported from VFS TextAsset
+```
+Usage: [arguments...] [-h|--help] [--version]
+Arguments:
+ [0] TextAsset/TableCfg directory
+ [1] output directory
+```
\ No newline at end of file