chizui/BeyondTools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Endfield??

Browse Source
This commit is contained in:
Chizuui
2025-11-27 06:42:57 +07:00
parent 7ef1566d15
commit 051cb4f3e8
22 changed files with 1813 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
BeyondTools.SparkBuffer/BeanType.cs Normal file
View File

@@ -0,0 +1,95 @@
using BeyondTools.SparkBuffer.Extensions;
namespace BeyondTools.SparkBuffer
{
public struct BeanType
{
public int typeHash;
public string name;
public Field[] fields;
public BeanType(BinaryReader reader)
{
typeHash = reader.ReadInt32();
name = reader.ReadSparkBufferString();
reader.Align4Bytes();
var fieldCount = reader.ReadInt32();
fields = new Field[fieldCount];
foreach (ref var field in fields.AsSpan())
{
field.name = reader.ReadSparkBufferString();
field.type = reader.ReadSparkType();
switch (field.type)
{
case SparkType.Bool:
case SparkType.Byte:
case SparkType.Int:
case SparkType.Long:
case SparkType.Float:
case SparkType.Double:
case SparkType.String:
break;
case SparkType.Enum:
case SparkType.Bean:
reader.Align4Bytes();
field.typeHash = reader.ReadInt32();
break;
case SparkType.Array:
field.type2 = reader.ReadSparkType();
if (field.type2.Value.IsEnumOrBeanType())
{
reader.Align4Bytes();
field.typeHash = reader.ReadInt32();
}
break;
case SparkType.Map:
field.type2 = reader.ReadSparkType();
field.type3 = reader.ReadSparkType();
if (field.type2.Value.IsEnumOrBeanType())
{
reader.Align4Bytes();
field.typeHash = reader.ReadInt32();
}
if (field.type3.Value.IsEnumOrBeanType())
{
reader.Align4Bytes();
field.typeHash2 = reader.ReadInt32();
}
break;
default:
throw new Exception(string.Format("Unsupported bean field type {0} at position {1}", field.type, reader.BaseStream.Position));
}
}
}
public struct Field
{
public string name;
public SparkType type;
/// <summary>
/// <see cref="SparkType.Array"/> or <see cref="SparkType.Map"/> key type
/// </summary>
public SparkType? type2;
/// <summary>
/// <see cref="SparkType.Map"/> value type
/// </summary>
public SparkType? type3;
/// <summary>
/// <see cref="SparkType.Bean"/>, <see cref="SparkType.Enum"/>, <see cref="SparkType.Array"/>, or <see cref="SparkType.Map"/> key type hash
/// </summary>
public int? typeHash;
/// <summary>
/// <see cref="SparkType.Map"/> value type hash
/// </summary>
public int? typeHash2;
}
}
}

16
BeyondTools.SparkBuffer/BeyondTools.SparkBuffer.csproj Normal file
View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ConsoleAppFramework" Version="5.3.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>

33
BeyondTools.SparkBuffer/EnumType.cs Normal file
View File

@@ -0,0 +1,33 @@
using BeyondTools.SparkBuffer.Extensions;
namespace BeyondTools.SparkBuffer
{
public struct EnumType
{
public int typeHash;
public string name;
public EnumItem[] enums;
public EnumType(BinaryReader reader)
{
typeHash = reader.ReadInt32();
name = reader.ReadSparkBufferString();
reader.Align4Bytes();
var enumCount = reader.ReadInt32();
enums = new EnumItem[enumCount];
foreach (ref var enumItem in enums.AsSpan())
{
enumItem.name = reader.ReadSparkBufferString();
reader.Align4Bytes();
enumItem.value = reader.ReadInt32();
}
}
public struct EnumItem
{
public string name;
public int value;
}
}
}

46
BeyondTools.SparkBuffer/Extensions/BinaryReaderExtensions.cs Normal file
View File

@@ -0,0 +1,46 @@
using System.IO;
using System.Text;
namespace BeyondTools.SparkBuffer.Extensions
{
public static class BinaryReaderExtensions
{
public static SparkType ReadSparkType(this BinaryReader reader)
=> (SparkType)reader.ReadByte();
public static long Seek(this BinaryReader reader, long pos, SeekOrigin seekOrigin = SeekOrigin.Begin)
=> reader.BaseStream.Seek(pos, seekOrigin);
public static string ReadSparkBufferString(this BinaryReader reader)
{
using MemoryStream buffer = new();
while (true)
{
byte b = reader.ReadByte();
if (b == 0)
break;
buffer.WriteByte(b);
}
return Encoding.UTF8.GetString(buffer.ToArray());
}
public static string ReadSparkBufferStringOffset(this BinaryReader reader)
{
var stringOffset = reader.ReadInt32();
if (stringOffset == -1)
return string.Empty;
var oldPosition = reader.BaseStream.Position;
reader.Seek(stringOffset);
var tmp = reader.ReadSparkBufferString();
reader.BaseStream.Position = oldPosition;
return tmp;
}
public static void Align4Bytes(this BinaryReader reader)
=> reader.Seek((reader.BaseStream.Position - 1) + (4 - ((reader.BaseStream.Position - 1) % 4)));
}
}

8
BeyondTools.SparkBuffer/Extensions/SparkTypeExtensions.cs Normal file
View File

@@ -0,0 +1,8 @@
namespace BeyondTools.SparkBuffer.Extensions
{
public static class SparkTypeExtensions
{
public static bool IsEnumOrBeanType(this SparkType type)
=> type is SparkType.Enum or SparkType.Bean;
}
}

8
BeyondTools.SparkBuffer/Properties/launchSettings.json Normal file
View File

@@ -0,0 +1,8 @@
{
"profiles": {
"BeyondTools.SparkBuffer": {
"commandName": "Project",
"commandLineArgs": "C:\\EndfieldOut\\Assets\\TextAsset\\TableCfg C:\\EndfieldOut\\TableCfgOutput"
}
}
}

243
BeyondTools.SparkBuffer/SparkBufferDump.cs Normal file
View File

@@ -0,0 +1,243 @@
using BeyondTools.SparkBuffer.Extensions;
using ConsoleAppFramework;
using System.Text.Json;
using System.Text.Json.Nodes;
namespace BeyondTools.SparkBuffer
{
internal class SparkBufferDump
{
static void Main(string[] args)
{
ConsoleApp.Run(args, (
[Argument] string tableCfgDir,
[Argument] string outputDir) =>
{
if (!Directory.Exists(tableCfgDir))
throw new FileNotFoundException($"{tableCfgDir} isn't a valid directory");
if (!Directory.Exists(outputDir))
Directory.CreateDirectory(outputDir);
var success = 0;
var processingCount = 0;
foreach (var tablePath in Directory.EnumerateFiles(tableCfgDir))
{
var fileName = Path.GetFileName(tablePath);
Console.WriteLine("Reading {0}...", fileName);
processingCount++;
using var file = File.OpenRead(tablePath);
using var binaryReader = new BinaryReader(file);
try
{
var typeDefOffset = binaryReader.ReadInt32();
var rootDefOffset = binaryReader.ReadInt32();
var dataOffset = binaryReader.ReadInt32();
binaryReader.Seek(typeDefOffset);
SparkManager.ReadTypeDefinitions(binaryReader);
binaryReader.Seek(rootDefOffset);
var rootDef = new BeanType.Field
{
type = binaryReader.ReadSparkType(),
name = binaryReader.ReadSparkBufferString()
};
if (rootDef.type.IsEnumOrBeanType())
{
binaryReader.Align4Bytes();
rootDef.typeHash = binaryReader.ReadInt32();
}
if (rootDef.type == SparkType.Map)
{
rootDef.type2 = binaryReader.ReadSparkType();
rootDef.type3 = binaryReader.ReadSparkType();
if (rootDef.type2.Value.IsEnumOrBeanType())
{
binaryReader.Align4Bytes();
rootDef.typeHash = binaryReader.ReadInt32();
}
if (rootDef.type3.Value.IsEnumOrBeanType())
{
binaryReader.Align4Bytes();
rootDef.typeHash2 = binaryReader.ReadInt32();
}
}
binaryReader.Seek(dataOffset);
var resultFilePath = Path.Combine(outputDir, $"{rootDef.name}.json");
switch (rootDef.type)
{
case SparkType.Bean:
var rootBeanType = SparkManager.BeanTypeFromHash((int)rootDef.typeHash!);
var beanDump = ReadBeanAsJObject(binaryReader, rootBeanType);
File.WriteAllText(resultFilePath, beanDump!.ToString());
break;
case SparkType.Map:
var mapDump = ReadMapAsJObject(binaryReader, rootDef);
File.WriteAllText(resultFilePath, JsonSerializer.Serialize(mapDump, SparkManager.jsonSerializerOptions));
break;
default:
throw new NotSupportedException(string.Format("Unsupported root type {0}", rootDef.type));
}
Console.WriteLine("Dumped {0} successfully", rootDef.name);
success++;
}
catch (Exception ex)
{
Console.WriteLine("Error in reading {0}, Error: {1}", fileName, ex.ToString());
}
}
Console.WriteLine("Dumped {0}/{1}", success, processingCount);
});
}
static JsonObject? ReadMapAsJObject(BinaryReader binaryReader, BeanType.Field typeDef)
{
var mapDump = new JsonObject();
var kvCount = binaryReader.ReadInt32();
for (int i = 0; i < kvCount; i++)
{
var key = typeDef.type2 switch
{
SparkType.String => binaryReader.ReadSparkBufferStringOffset(),
SparkType.Int => binaryReader.ReadInt32().ToString(),
_ => throw new NotSupportedException(string.Format("Unsupported map key type {0}", typeDef.type2)),
};
mapDump[key] = null;
}
for (int i = 0; i < kvCount; i++)
{
mapDump[i] = typeDef.type3 switch
{
SparkType.Bean => ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)typeDef.typeHash2!), true),
SparkType.String => binaryReader.ReadSparkBufferStringOffset(),
SparkType.Int => binaryReader.ReadInt32(),
_ => throw new NotSupportedException(string.Format("Unsupported map value type {0}", typeDef.type3)),
};
}
return mapDump;
}
static JsonObject? ReadBeanAsJObject(BinaryReader binaryReader, BeanType beanType, bool pointer = false)
{
long? pointerOrigin = null;
if (pointer)
{
var beanOffset = binaryReader.ReadInt32();
if (beanOffset == -1)
return null;
pointerOrigin = binaryReader.BaseStream.Position;
binaryReader.Seek(beanOffset);
}
var dumpObj = new JsonObject();
foreach (var (fieldIndex, beanField) in beanType.fields.Index())
{
long? origin = null;
if (beanField.type == SparkType.Array)
{
var fieldOffset = binaryReader.ReadInt32();
if (fieldOffset == -1)
{
dumpObj[beanField.name] = null;
continue;
}
origin = binaryReader.BaseStream.Position;
binaryReader.Seek(fieldOffset);
}
switch (beanField.type)
{
case SparkType.Array:
var jArray = new JsonArray();
var itemCount = binaryReader.ReadInt32();
while (itemCount-- > 0)
{
switch (beanField.type2)
{
case SparkType.String:
jArray.Add(binaryReader.ReadSparkBufferStringOffset());
break;
case SparkType.Bean:
jArray.Add(ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true));
break;
case SparkType.Float:
jArray.Add(binaryReader.ReadSingle());
break;
case SparkType.Long:
jArray.Add(binaryReader.ReadInt64());
break;
case SparkType.Int:
case SparkType.Enum:
jArray.Add(binaryReader.ReadInt32());
break;
case SparkType.Bool:
jArray.Add(binaryReader.ReadBoolean());
break;
default:
throw new NotSupportedException(string.Format("Unsupported array type {0} on bean field, position: {1}", beanField.type2, binaryReader.BaseStream.Position));
}
}
dumpObj[beanField.name] = jArray;
break;
case SparkType.Int:
case SparkType.Enum:
dumpObj[beanField.name] = binaryReader.ReadInt32();
break;
case SparkType.Long:
dumpObj[beanField.name] = binaryReader.ReadInt64();
break;
case SparkType.Float:
dumpObj[beanField.name] = binaryReader.ReadSingle();
break;
case SparkType.Double:
dumpObj[beanField.name] = binaryReader.ReadDouble();
break;
case SparkType.String:
dumpObj[beanField.name] = binaryReader.ReadSparkBufferStringOffset();
break;
case SparkType.Bean:
dumpObj[beanField.name] = ReadBeanAsJObject(binaryReader, SparkManager.BeanTypeFromHash((int)beanField.typeHash!), true);
break;
case SparkType.Bool:
dumpObj[beanField.name] = binaryReader.ReadBoolean();
if (beanType.fields.Length > fieldIndex + 1 && beanType.fields[fieldIndex + 1].type != SparkType.Bool)
binaryReader.Align4Bytes();
break;
case SparkType.Map:
var mapOffset = binaryReader.ReadInt32();
var mapOrigin = binaryReader.BaseStream.Position;
binaryReader.Seek(mapOffset);
dumpObj[beanField.name] = ReadMapAsJObject(binaryReader, beanField);
binaryReader.Seek(mapOrigin);
break;
case SparkType.Byte:
throw new Exception(string.Format("Dumping bean field type {0} isn't supported, position: {1}", beanField.type, binaryReader.BaseStream.Position));
}
if (origin is not null)
binaryReader.BaseStream.Position = (long)origin;
}
if (pointerOrigin is not null)
binaryReader.BaseStream.Position = (long)pointerOrigin;
return dumpObj;
}
}
}

38
BeyondTools.SparkBuffer/SparkManager.cs Normal file
View File

@@ -0,0 +1,38 @@
using BeyondTools.SparkBuffer.Extensions;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace BeyondTools.SparkBuffer
{
public static class SparkManager
{
public static readonly JsonSerializerOptions jsonSerializerOptions = new() { IncludeFields = true, WriteIndented = true, NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals };
private static readonly Dictionary<int, BeanType> beanTypeMap = [];
private static readonly Dictionary<int, EnumType> enumTypeMap = [];
public static BeanType BeanTypeFromHash(int hash)
=> beanTypeMap[hash];
public static void ReadTypeDefinitions(BinaryReader reader)
{
var typeDefCount = reader.ReadInt32();
while (typeDefCount-- > 0)
{
var sparkType = reader.ReadSparkType();
reader.Align4Bytes();
if (sparkType == SparkType.Enum)
{
var enumType = new EnumType(reader);
enumTypeMap.TryAdd(enumType.typeHash, enumType);
}
else if (sparkType == SparkType.Bean)
{
var beanType = new BeanType(reader);
beanTypeMap.TryAdd(beanType.typeHash, beanType);
}
}
}
}
}

17
BeyondTools.SparkBuffer/SparkType.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace BeyondTools.SparkBuffer
{
public enum SparkType : byte
{
Bool,
Byte,
Int,
Long,
Float,
Double,
Enum,
String,
Bean,
Array,
Map
}
}

18
BeyondTools.VFS/BeyondTools.VFS.csproj Normal file
View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ConsoleAppFramework" Version="5.3.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="System.IO.Hashing" Version="9.0.0" />
</ItemGroup>
</Project>

930
BeyondTools.VFS/Crypto/CSChaCha20.cs Normal file
View File

@@ -0,0 +1,930 @@
/*
* Copyright (c) 2015, 2018 Scott Bennett
* (c) 2018-2023 Kaarlo Räihä
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
using System;
using System.IO;
using System.Threading.Tasks;
using System.Runtime.Intrinsics;
using System.Runtime.CompilerServices;
namespace BeyondTools.VFS.Crypto;
/// <summary>
/// Chosen SIMD mode
/// </summary>
public enum SimdMode
{
/// <summary>
/// Autodetect
/// </summary>
AutoDetect = 0,
/// <summary>
/// 128 bit SIMD
/// </summary>
V128,
/// <summary>
/// 256 bit SIMD
/// </summary>
V256,
/// <summary>
/// 512 bit SIMD
/// </summary>
V512,
/// <summary>
/// No SIMD
/// </summary>
None
}
/// <summary>
/// Class for ChaCha20 encryption / decryption
/// </summary>
public sealed class CSChaCha20 : IDisposable
{
/// <summary>
/// Only allowed key lenght in bytes
/// </summary>
public const int allowedKeyLength = 32;
/// <summary>
/// Only allowed nonce lenght in bytes
/// </summary>
public const int allowedNonceLength = 12;
/// <summary>
/// How many bytes are processed per loop
/// </summary>
public const int processBytesAtTime = 64;
private const int stateLength = 16;
/// <summary>
/// The ChaCha20 state (aka "context")
/// </summary>
private readonly uint[] state = new uint[stateLength];
/// <summary>
/// Determines if the objects in this class have been disposed of. Set to true by the Dispose() method.
/// </summary>
private bool isDisposed = false;
/// <summary>
/// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens.
/// </summary>
/// <remarks>
/// See <a href="https://tools.ietf.org/html/rfc7539#page-10">ChaCha20 Spec Section 2.4</a> for a detailed description of the inputs.
/// </remarks>
/// <param name="key">
/// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers
/// </param>
/// <param name="nonce">
/// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers
/// </param>
/// <param name="counter">
/// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer
/// </param>
public CSChaCha20(byte[] key, byte[] nonce, uint counter)
{
KeySetup(key);
IvSetup(nonce, counter);
}
/// <summary>
/// Set up a new ChaCha20 state. The lengths of the given parameters are checked before encryption happens.
/// </summary>
/// <remarks>
/// See <a href="https://tools.ietf.org/html/rfc7539#page-10">ChaCha20 Spec Section 2.4</a> for a detailed description of the inputs.
/// </remarks>
/// <param name="key">A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers</param>
/// <param name="nonce">A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers</param>
/// <param name="counter">A 4-byte (32-bit) block counter, treated as a 32-bit little-endian unsigned integer</param>
public CSChaCha20(ReadOnlySpan<byte> key, ReadOnlySpan<byte> nonce, uint counter)
{
KeySetup(key.ToArray());
IvSetup(nonce.ToArray(), counter);
}
/// <summary>
/// The ChaCha20 state (aka "context"). Read-Only.
/// </summary>
public uint[] State
{
get
{
return state;
}
}
// These are the same constants defined in the reference implementation.
// http://cr.yp.to/streamciphers/timings/estreambench/submissions/salsa20/chacha8/ref/chacha.c
private static readonly byte[] sigma = "expand 32-byte k"u8.ToArray();
private static readonly byte[] tau = "expand 16-byte k"u8.ToArray();
/// <summary>
/// Set up the ChaCha state with the given key. A 32-byte key is required and enforced.
/// </summary>
/// <param name="key">
/// A 32-byte (256-bit) key, treated as a concatenation of eight 32-bit little-endian integers
/// </param>
private void KeySetup(byte[] key)
{
if (key == null)
{
throw new ArgumentNullException("Key is null");
}
if (key.Length != allowedKeyLength)
{
throw new ArgumentException($"Key length must be {allowedKeyLength}. Actual: {key.Length}");
}
state[4] = Util.U8To32Little(key, 0);
state[5] = Util.U8To32Little(key, 4);
state[6] = Util.U8To32Little(key, 8);
state[7] = Util.U8To32Little(key, 12);
byte[] constants = key.Length == allowedKeyLength ? sigma : tau;
int keyIndex = key.Length - 16;
state[8] = Util.U8To32Little(key, keyIndex + 0);
state[9] = Util.U8To32Little(key, keyIndex + 4);
state[10] = Util.U8To32Little(key, keyIndex + 8);
state[11] = Util.U8To32Little(key, keyIndex + 12);
state[0] = Util.U8To32Little(constants, 0);
state[1] = Util.U8To32Little(constants, 4);
state[2] = Util.U8To32Little(constants, 8);
state[3] = Util.U8To32Little(constants, 12);
}
/// <summary>
/// Set up the ChaCha state with the given nonce (aka Initialization Vector or IV) and block counter. A 12-byte nonce and a 4-byte counter are required.
/// </summary>
/// <param name="nonce">
/// A 12-byte (96-bit) nonce, treated as a concatenation of three 32-bit little-endian integers
/// </param>
/// <param name="counter">
/// A 4-byte (32-bit) block counter, treated as a 32-bit little-endian integer
/// </param>
private void IvSetup(byte[] nonce, uint counter)
{
if (nonce == null)
{
// There has already been some state set up. Clear it before exiting.
Dispose();
throw new ArgumentNullException("Nonce is null");
}
if (nonce.Length != allowedNonceLength)
{
// There has already been some state set up. Clear it before exiting.
Dispose();
throw new ArgumentException($"Nonce length must be {allowedNonceLength}. Actual: {nonce.Length}");
}
state[12] = counter;
state[13] = Util.U8To32Little(nonce, 0);
state[14] = Util.U8To32Little(nonce, 4);
state[15] = Util.U8To32Little(nonce, 8);
}
private static SimdMode DetectSimdMode()
{
if (Vector512.IsHardwareAccelerated)
{
return SimdMode.V512;
}
else if (Vector256.IsHardwareAccelerated)
{
return SimdMode.V256;
}
else if (Vector128.IsHardwareAccelerated)
{
return SimdMode.V128;
}
return SimdMode.None;
}
#region Encryption methods
/// <summary>
/// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="output">Output byte array, must have enough bytes</param>
/// <param name="input">Input byte array</param>
/// <param name="numBytes">Number of bytes to encrypt</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void EncryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
{
if (output == null)
{
throw new ArgumentNullException("output", "Output cannot be null");
}
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (numBytes < 0 || numBytes > input.Length)
{
throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
}
if (output.Length < numBytes)
{
throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkBytes(output, input, numBytes, simdMode);
}
/// <summary>
/// Encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
/// </summary>
/// <param name="output">Output stream</param>
/// <param name="input">Input stream</param>
/// <param name="howManyBytesToProcessAtTime">How many bytes to read and write at time, default is 1024</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void EncryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
{
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime);
}
/// <summary>
/// Async encrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
/// </summary>
/// <param name="output">Output stream</param>
/// <param name="input">Input stream</param>
/// <param name="howManyBytesToProcessAtTime">How many bytes to read and write at time, default is 1024</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public async Task EncryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
{
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime);
}
/// <summary>
/// Encrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="output">Output byte array, must have enough bytes</param>
/// <param name="input">Input byte array</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void EncryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (output == null)
{
throw new ArgumentNullException("output", "Output cannot be null");
}
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkBytes(output, input, input.Length, simdMode);
}
/// <summary>
/// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="input">Input byte array</param>
/// <param name="numBytes">Number of bytes to encrypt</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains encrypted bytes</returns>
public byte[] EncryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (numBytes < 0 || numBytes > input.Length)
{
throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] returnArray = new byte[numBytes];
WorkBytes(returnArray, input, numBytes, simdMode);
return returnArray;
}
/// <summary>
/// Encrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="input">Input byte array</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains encrypted bytes</returns>
public byte[] EncryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] returnArray = new byte[input.Length];
WorkBytes(returnArray, input, input.Length, simdMode);
return returnArray;
}
/// <summary>
/// Encrypt string as UTF8 byte array, returns byte array that is allocated by method.
/// </summary>
/// <remarks>Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform</remarks>
/// <param name="input">Input string</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains encrypted bytes</returns>
public byte[] EncryptString(string input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] utf8Bytes = System.Text.Encoding.UTF8.GetBytes(input);
byte[] returnArray = new byte[utf8Bytes.Length];
WorkBytes(returnArray, utf8Bytes, utf8Bytes.Length, simdMode);
return returnArray;
}
#endregion // Encryption methods
#region // Decryption methods
/// <summary>
/// Decrypt arbitrary-length byte array (input), writing the resulting byte array to the output buffer.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="output">Output byte array</param>
/// <param name="input">Input byte array</param>
/// <param name="numBytes">Number of bytes to decrypt</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void DecryptBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
{
if (output == null)
{
throw new ArgumentNullException("output", "Output cannot be null");
}
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (numBytes < 0 || numBytes > input.Length)
{
throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
}
if (output.Length < numBytes)
{
throw new ArgumentOutOfRangeException("output", $"Output byte array should be able to take at least {numBytes}");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkBytes(output, input, numBytes, simdMode);
}
/// <summary>
/// Decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
/// </summary>
/// <param name="output">Output stream</param>
/// <param name="input">Input stream</param>
/// <param name="howManyBytesToProcessAtTime">How many bytes to read and write at time, default is 1024</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void DecryptStream(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
{
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkStreams(output, input, simdMode, howManyBytesToProcessAtTime);
}
/// <summary>
/// Async decrypt arbitrary-length byte stream (input), writing the resulting bytes to another stream (output)
/// </summary>
/// <param name="output">Output stream</param>
/// <param name="input">Input stream</param>
/// <param name="howManyBytesToProcessAtTime">How many bytes to read and write at time, default is 1024</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public async Task DecryptStreamAsync(Stream output, Stream input, int howManyBytesToProcessAtTime = 1024, SimdMode simdMode = SimdMode.AutoDetect)
{
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
await WorkStreamsAsync(output, input, simdMode, howManyBytesToProcessAtTime);
}
/// <summary>
/// Decrypt arbitrary-length byte array (input), writing the resulting byte array to preallocated output buffer.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="output">Output byte array, must have enough bytes</param>
/// <param name="input">Input byte array</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
public void DecryptBytes(byte[] output, byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (output == null)
{
throw new ArgumentNullException("output", "Output cannot be null");
}
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
WorkBytes(output, input, input.Length, simdMode);
}
/// <summary>
/// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="input">Input byte array</param>
/// <param name="numBytes">Number of bytes to encrypt</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains decrypted bytes</returns>
public byte[] DecryptBytes(byte[] input, int numBytes, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (numBytes < 0 || numBytes > input.Length)
{
throw new ArgumentOutOfRangeException("numBytes", "The number of bytes to read must be between [0..input.Length]");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] returnArray = new byte[numBytes];
WorkBytes(returnArray, input, numBytes, simdMode);
return returnArray;
}
/// <summary>
/// Decrypt arbitrary-length byte array (input), writing the resulting byte array that is allocated by method.
/// </summary>
/// <remarks>Since this is symmetric operation, it doesn't really matter if you use Encrypt or Decrypt method</remarks>
/// <param name="input">Input byte array</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains decrypted bytes</returns>
public byte[] DecryptBytes(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] returnArray = new byte[input.Length];
WorkBytes(returnArray, input, input.Length, simdMode);
return returnArray;
}
/// <summary>
/// Decrypt UTF8 byte array to string.
/// </summary>
/// <remarks>Here you can NOT swap encrypt and decrypt methods, because of bytes-string transform</remarks>
/// <param name="input">Byte array</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
/// <returns>Byte array that contains encrypted bytes</returns>
public string DecryptUTF8ByteArray(byte[] input, SimdMode simdMode = SimdMode.AutoDetect)
{
if (input == null)
{
throw new ArgumentNullException("input", "Input cannot be null");
}
if (simdMode == SimdMode.AutoDetect)
{
simdMode = DetectSimdMode();
}
byte[] tempArray = new byte[input.Length];
WorkBytes(tempArray, input, input.Length, simdMode);
return System.Text.Encoding.UTF8.GetString(tempArray);
}
#endregion // Decryption methods
private void WorkStreams(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024)
{
int readBytes;
byte[] inputBuffer = new byte[howManyBytesToProcessAtTime];
byte[] outputBuffer = new byte[howManyBytesToProcessAtTime];
while ((readBytes = input.Read(inputBuffer, 0, howManyBytesToProcessAtTime)) > 0)
{
// Encrypt or decrypt
WorkBytes(output: outputBuffer, input: inputBuffer, numBytes: readBytes, simdMode);
// Write buffer
output.Write(outputBuffer, 0, readBytes);
}
}
private async Task WorkStreamsAsync(Stream output, Stream input, SimdMode simdMode, int howManyBytesToProcessAtTime = 1024)
{
byte[] readBytesBuffer = new byte[howManyBytesToProcessAtTime];
byte[] writeBytesBuffer = new byte[howManyBytesToProcessAtTime];
int howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime);
while (howManyBytesWereRead > 0)
{
// Encrypt or decrypt
WorkBytes(output: writeBytesBuffer, input: readBytesBuffer, numBytes: howManyBytesWereRead, simdMode);
// Write
await output.WriteAsync(writeBytesBuffer, 0, howManyBytesWereRead);
// Read more
howManyBytesWereRead = await input.ReadAsync(readBytesBuffer, 0, howManyBytesToProcessAtTime);
}
}
/// <summary>
/// Encrypt or decrypt an arbitrary-length byte array (input), writing the resulting byte array to the output buffer. The number of bytes to read from the input buffer is determined by numBytes.
/// </summary>
/// <param name="output">Output byte array</param>
/// <param name="input">Input byte array</param>
/// <param name="numBytes">How many bytes to process</param>
/// <param name="simdMode">Chosen SIMD mode (default is auto-detect)</param>
private void WorkBytes(byte[] output, byte[] input, int numBytes, SimdMode simdMode)
{
if (isDisposed)
{
throw new ObjectDisposedException("state", "The ChaCha state has been disposed");
}
uint[] x = new uint[stateLength]; // Working buffer
byte[] tmp = new byte[processBytesAtTime]; // Temporary buffer
int offset = 0;
int howManyFullLoops = numBytes / processBytesAtTime;
int tailByteCount = numBytes - howManyFullLoops * processBytesAtTime;
for (int loop = 0; loop < howManyFullLoops; loop++)
{
UpdateStateAndGenerateTemporaryBuffer(state, x, tmp);
if (simdMode == SimdMode.V512)
{
// 1 x 64 bytes
Vector512<byte> inputV = Vector512.Create(input, offset);
Vector512<byte> tmpV = Vector512.Create(tmp, 0);
Vector512<byte> outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset);
}
else if (simdMode == SimdMode.V256)
{
// 2 x 32 bytes
Vector256<byte> inputV = Vector256.Create(input, offset);
Vector256<byte> tmpV = Vector256.Create(tmp, 0);
Vector256<byte> outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset);
inputV = Vector256.Create(input, offset + 32);
tmpV = Vector256.Create(tmp, 32);
outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset + 32);
}
else if (simdMode == SimdMode.V128)
{
// 4 x 16 bytes
Vector128<byte> inputV = Vector128.Create(input, offset);
Vector128<byte> tmpV = Vector128.Create(tmp, 0);
Vector128<byte> outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset);
inputV = Vector128.Create(input, offset + 16);
tmpV = Vector128.Create(tmp, 16);
outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset + 16);
inputV = Vector128.Create(input, offset + 32);
tmpV = Vector128.Create(tmp, 32);
outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset + 32);
inputV = Vector128.Create(input, offset + 48);
tmpV = Vector128.Create(tmp, 48);
outputV = inputV ^ tmpV;
outputV.CopyTo(output, offset + 48);
}
else
{
for (int i = 0; i < processBytesAtTime; i += 4)
{
// Small unroll
int start = i + offset;
output[start] = (byte)(input[start] ^ tmp[i]);
output[start + 1] = (byte)(input[start + 1] ^ tmp[i + 1]);
output[start + 2] = (byte)(input[start + 2] ^ tmp[i + 2]);
output[start + 3] = (byte)(input[start + 3] ^ tmp[i + 3]);
}
}
offset += processBytesAtTime;
}
// In case there are some bytes left
if (tailByteCount > 0)
{
UpdateStateAndGenerateTemporaryBuffer(state, x, tmp);
for (int i = 0; i < tailByteCount; i++)
{
output[i + offset] = (byte)(input[i + offset] ^ tmp[i]);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void UpdateStateAndGenerateTemporaryBuffer(uint[] stateToModify, uint[] workingBuffer, byte[] temporaryBuffer)
{
// Copy state to working buffer
Buffer.BlockCopy(stateToModify, 0, workingBuffer, 0, stateLength * sizeof(uint));
for (int i = 0; i < 10; i++)
{
QuarterRound(workingBuffer, 0, 4, 8, 12);
QuarterRound(workingBuffer, 1, 5, 9, 13);
QuarterRound(workingBuffer, 2, 6, 10, 14);
QuarterRound(workingBuffer, 3, 7, 11, 15);
QuarterRound(workingBuffer, 0, 5, 10, 15);
QuarterRound(workingBuffer, 1, 6, 11, 12);
QuarterRound(workingBuffer, 2, 7, 8, 13);
QuarterRound(workingBuffer, 3, 4, 9, 14);
}
for (int i = 0; i < stateLength; i++)
{
Util.ToBytes(temporaryBuffer, Util.Add(workingBuffer[i], stateToModify[i]), 4 * i);
}
stateToModify[12] = Util.AddOne(stateToModify[12]);
if (stateToModify[12] <= 0)
{
/* Stopping at 2^70 bytes per nonce is the user's responsibility */
stateToModify[13] = Util.AddOne(stateToModify[13]);
}
}
/// <summary>
/// The ChaCha Quarter Round operation. It operates on four 32-bit unsigned integers within the given buffer at indices a, b, c, and d.
/// </summary>
/// <remarks>
/// The ChaCha state does not have four integer numbers: it has 16. So the quarter-round operation works on only four of them -- hence the name. Each quarter round operates on four predetermined numbers in the ChaCha state.
/// See <a href="https://tools.ietf.org/html/rfc7539#page-4">ChaCha20 Spec Sections 2.1 - 2.2</a>.
/// </remarks>
/// <param name="x">A ChaCha state (vector). Must contain 16 elements.</param>
/// <param name="a">Index of the first number</param>
/// <param name="b">Index of the second number</param>
/// <param name="c">Index of the third number</param>
/// <param name="d">Index of the fourth number</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void QuarterRound(uint[] x, uint a, uint b, uint c, uint d)
{
x[a] = Util.Add(x[a], x[b]);
x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 16);
x[c] = Util.Add(x[c], x[d]);
x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 12);
x[a] = Util.Add(x[a], x[b]);
x[d] = Util.Rotate(Util.XOr(x[d], x[a]), 8);
x[c] = Util.Add(x[c], x[d]);
x[b] = Util.Rotate(Util.XOr(x[b], x[c]), 7);
}
#region Destructor and Disposer
/// <summary>
/// Clear and dispose of the internal state. The finalizer is only called if Dispose() was never called on this cipher.
/// </summary>
~CSChaCha20()
{
Dispose(false);
}
/// <summary>
/// Clear and dispose of the internal state. Also request the GC not to call the finalizer, because all cleanup has been taken care of.
/// </summary>
public void Dispose()
{
Dispose(true);
/*
* The Garbage Collector does not need to invoke the finalizer because Dispose(bool) has already done all the cleanup needed.
*/
GC.SuppressFinalize(this);
}
/// <summary>
/// This method should only be invoked from Dispose() or the finalizer. This handles the actual cleanup of the resources.
/// </summary>
/// <param name="disposing">
/// Should be true if called by Dispose(); false if called by the finalizer
/// </param>
private void Dispose(bool disposing)
{
if (!isDisposed)
{
if (disposing)
{
/* Cleanup managed objects by calling their Dispose() methods */
}
/* Cleanup any unmanaged objects here */
Array.Clear(state, 0, stateLength);
}
isDisposed = true;
}
#endregion // Destructor and Disposer
}
/// <summary>
/// Utilities that are used during compression
/// </summary>
public static class Util
{
/// <summary>
/// n-bit left rotation operation (towards the high bits) for 32-bit integers.
/// </summary>
/// <param name="v"></param>
/// <param name="c"></param>
/// <returns>The result of (v LEFTSHIFT c)</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint Rotate(uint v, int c)
{
unchecked
{
return v << c | v >> 32 - c;
}
}
/// <summary>
/// Unchecked integer exclusive or (XOR) operation.
/// </summary>
/// <param name="v"></param>
/// <param name="w"></param>
/// <returns>The result of (v XOR w)</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint XOr(uint v, uint w)
{
return unchecked(v ^ w);
}
/// <summary>
/// Unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32.
/// </summary>
/// <remarks>
/// See <a href="https://tools.ietf.org/html/rfc7539#page-4">ChaCha20 Spec Section 2.1</a>.
/// </remarks>
/// <param name="v"></param>
/// <param name="w"></param>
/// <returns>The result of (v + w) modulo 2^32</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint Add(uint v, uint w)
{
return unchecked(v + w);
}
/// <summary>
/// Add 1 to the input parameter using unchecked integer addition. The ChaCha spec defines certain operations to use 32-bit unsigned integer addition modulo 2^32.
/// </summary>
/// <remarks>
/// See <a href="https://tools.ietf.org/html/rfc7539#page-4">ChaCha20 Spec Section 2.1</a>.
/// </remarks>
/// <param name="v"></param>
/// <returns>The result of (v + 1) modulo 2^32</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint AddOne(uint v)
{
return unchecked(v + 1);
}
/// <summary>
/// Convert four bytes of the input buffer into an unsigned 32-bit integer, beginning at the inputOffset.
/// </summary>
/// <param name="p"></param>
/// <param name="inputOffset"></param>
/// <returns>An unsigned 32-bit integer</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint U8To32Little(byte[] p, int inputOffset)
{
unchecked
{
return p[inputOffset]
| (uint)p[inputOffset + 1] << 8
| (uint)p[inputOffset + 2] << 16
| (uint)p[inputOffset + 3] << 24;
}
}
/// <summary>
/// Serialize the input integer into the output buffer. The input integer will be split into 4 bytes and put into four sequential places in the output buffer, starting at the outputOffset.
/// </summary>
/// <param name="output"></param>
/// <param name="input"></param>
/// <param name="outputOffset"></param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ToBytes(byte[] output, uint input, int outputOffset)
{
unchecked
{
output[outputOffset] = (byte)input;
output[outputOffset + 1] = (byte)(input >> 8);
output[outputOffset + 2] = (byte)(input >> 16);
output[outputOffset + 3] = (byte)(input >> 24);
}
}
}

20
BeyondTools.VFS/EVFSBlockType.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace BeyondTools.VFS
{
public enum EVFSBlockType : byte
{
All,
InitialAudio = 1,
InitialBundle,
BundleManifest,
LowShader,
Audio = 11,
Bundle,
TextAsset = 14,
Video,
IV,
Streaming,
IFixPatch = 21,
Raw = 31
}
}

13
BeyondTools.VFS/EVFSLoaderPosType.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace BeyondTools.VFS
{
public enum EVFSLoaderPosType : byte
{
None,
PersistAsset,
StreamAsset,
VFS = 10,
VFS_PersistAsset,
VFS_StreamAsset,
VFS_Build
}
}

33
BeyondTools.VFS/Extensions/StreamExtensions.cs Normal file
View File

@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BeyondTools.VFS.Extensions
{
internal static class StreamExtensions
{
public static void CopyBytes(this Stream inStream, Stream outStream, long? count = null)
{
if (count == null)
{
inStream.CopyTo(outStream);
return;
}
long readBytes = 0L;
var buffer = new byte[64 * 1024];
do
{
var toRead = Math.Min((long)(count - readBytes), buffer.LongLength);
var readNow = inStream.Read(buffer, 0, (int)toRead);
if (readNow == 0)
break;
outStream.Write(buffer, 0, readNow);
readBytes += readNow;
} while (readBytes < count);
}
}
}

16
BeyondTools.VFS/FVFBlockChunkInfo.cs Normal file
View File

@@ -0,0 +1,16 @@
using System.Text.Json.Serialization;
namespace BeyondTools.VFS
{
public struct FVFBlockChunkInfo
{
public const string FILE_EXTENSION = ".chk";
public UInt128 md5Name;
public UInt128 contentMD5;
public long length;
public EVFSBlockType blockType;
[JsonIgnore]
public FVFBlockFileInfo[] files;
}
}

17
BeyondTools.VFS/FVFBlockFileInfo.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace BeyondTools.VFS
{
public struct FVFBlockFileInfo
{
public string fileName;
public long fileNameHash;
public UInt128 fileChunkMD5Name;
public UInt128 fileDataMD5;
public long offset;
public long len;
public EVFSBlockType blockType;
public bool bUseEncrypt;
public long ivSeed;
public bool bIsDirect;
public EVFSLoaderPosType loaderPosType;
}
}

8
BeyondTools.VFS/Properties/launchSettings.json Normal file
View File

@@ -0,0 +1,8 @@
{
"profiles": {
"BeyondTools.VFS": {
"commandName": "Project",
"commandLineArgs": "C:/Users/rafi1/Desktop/etc/Beyond_TMP/Game/Beyond_Data/StreamingAssets InitialBundle C:/EndfieldOut/Assets"
}
}
}

95
BeyondTools.VFS/VFBlockMainInfo.cs Normal file
View File

@@ -0,0 +1,95 @@
using System.Buffers.Binary;
using System.Runtime.InteropServices;
using System.Text;
namespace BeyondTools.VFS
{
public class VFBlockMainInfo
{
public VFBlockMainInfo(byte[] bytes, int offset = 0)
{
version = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
offset += sizeof(int);
// TODO: CRC and stuff idk
offset += 12;
ushort groupCfgNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset));
offset += sizeof(ushort);
groupCfgName = Encoding.UTF8.GetString(bytes.AsSpan(offset, groupCfgNameLength));
offset += groupCfgNameLength;
groupCfgHashName = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
groupFileInfoNum = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
offset += sizeof(int);
groupChunksLength = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
blockType = (EVFSBlockType)bytes[offset++];
var chunkCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
allChunks = GC.AllocateUninitializedArray<FVFBlockChunkInfo>(chunkCount);
offset += sizeof(int);
foreach (ref var chunk in allChunks.AsSpan())
{
chunk.md5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
offset += Marshal.SizeOf<UInt128>();
chunk.contentMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
offset += Marshal.SizeOf<UInt128>();
chunk.length = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
chunk.blockType = (EVFSBlockType)bytes[offset++];
var fileCount = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset));
chunk.files = GC.AllocateUninitializedArray<FVFBlockFileInfo>(fileCount);
offset += sizeof(int);
foreach (ref var file in chunk.files.AsSpan())
{
ushort fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(offset));
offset += sizeof(ushort);
file.fileName = Encoding.UTF8.GetString(bytes.AsSpan(offset, fileNameLength));
offset += fileNameLength;
file.fileNameHash = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
file.fileChunkMD5Name = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
offset += Marshal.SizeOf<UInt128>();
file.fileDataMD5 = BinaryPrimitives.ReadUInt128LittleEndian(bytes.AsSpan(offset));
offset += Marshal.SizeOf<UInt128>();
file.offset = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
file.len = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
file.blockType = (EVFSBlockType)bytes[offset++];
file.bUseEncrypt = Convert.ToBoolean(bytes[offset++]);
if (file.bUseEncrypt)
{
file.ivSeed = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(offset));
offset += sizeof(long);
}
}
}
}
public int version;
public string groupCfgName;
public long groupCfgHashName;
public int groupFileInfoNum;
public long groupChunksLength;
public EVFSBlockType blockType;
public FVFBlockChunkInfo[] allChunks;
}
}

11
BeyondTools.VFS/VFSDefine.cs Normal file
View File

@@ -0,0 +1,11 @@
namespace BeyondTools.VFS
{
public static class VFSDefine
{
public const string CHACHA_KEY = "eU1cu+MYQiaYdVherRzV86pv/N/lIU/9gIk+5n5Vj4Y=";
public const string VFS_DIR = "VFS";
public const int VFS_PROTO_VERSION = 3;
public const int VFS_VFB_HEAD_LEN = 16;
public const int BLOCK_HEAD_LEN = 12;
}
}

103
BeyondTools.VFS/VFSDump.cs Normal file
View File

@@ -0,0 +1,103 @@
using BeyondTools.VFS.Crypto;
using BeyondTools.VFS.Extensions;
using ConsoleAppFramework;
using System.IO.Hashing;
using System.Text;
namespace BeyondTools.VFS;
internal class VFSDump
{
static readonly Dictionary<EVFSBlockType, string> blockTypeMap = new()
{
{ EVFSBlockType.Audio, "MainAudio" },
{ EVFSBlockType.Bundle, "MainBundles" },
{ EVFSBlockType.BundleManifest, "BundleManifest" },
{ EVFSBlockType.IFixPatch, "IFixPatchOut" },
{ EVFSBlockType.InitialAudio, "InitAudio" },
{ EVFSBlockType.InitialBundle, "InitBundles" },
{ EVFSBlockType.IV, "IV" },
{ EVFSBlockType.LowShader, "LowShader" },
// { EVFSBlockType.Raw, "" }, Not present
{ EVFSBlockType.Streaming, "Streaming" },
{ EVFSBlockType.TextAsset, "TextAsset" },
{ EVFSBlockType.Video, "Video" },
};
static void Main(string[] args)
{
ConsoleApp.Run(args, (
[Argument] string streamingAssetsPath,
[Argument] EVFSBlockType dumpAssetType = EVFSBlockType.All,
[Argument] string? outputDir = null) =>
{
streamingAssetsPath = Path.Combine(streamingAssetsPath, VFSDefine.VFS_DIR);
outputDir ??= Path.Combine(AppContext.BaseDirectory, "Assets");
if (dumpAssetType == EVFSBlockType.All)
{
foreach (var type in blockTypeMap.Keys)
{
DumpAssetByType(streamingAssetsPath, type, outputDir);
}
}
else
{
DumpAssetByType(streamingAssetsPath, dumpAssetType, outputDir);
}
});
}
private static void DumpAssetByType(string streamingAssetsPath, EVFSBlockType dumpAssetType, string outputDir)
{
Console.WriteLine("Dumping {0} files...", dumpAssetType.ToString());
// TODO: This is a temporary solution that makes this thing only worked on some EVFSBlockType, i haven't been able to figure out Crc32Utils.UnityCRC64
var blockDir = Directory.EnumerateDirectories(streamingAssetsPath).First(x => x.Split('/', '\\').Last().StartsWith(Convert.ToHexString(Crc32.Hash(Encoding.UTF8.GetBytes(blockTypeMap[dumpAssetType])))));
var blockFilePath = Path.Combine(blockDir, blockDir.Split('/', '\\').Last() + ".blc");
var blockFile = File.ReadAllBytes(blockFilePath);
byte[] nonce = GC.AllocateUninitializedArray<byte>(VFSDefine.BLOCK_HEAD_LEN);
Buffer.BlockCopy(blockFile, 0, nonce, 0, nonce.Length);
var chacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), nonce, 1);
var decryptedBytes = chacha.DecryptBytes(blockFile[VFSDefine.BLOCK_HEAD_LEN..]);
Buffer.BlockCopy(decryptedBytes, 0, blockFile, VFSDefine.BLOCK_HEAD_LEN, decryptedBytes.Length);
var vfBlockMainInfo = new VFBlockMainInfo(blockFile);
foreach (var chunk in vfBlockMainInfo.allChunks)
{
var chunkMd5Name = Convert.ToHexString(BitConverter.GetBytes(chunk.md5Name)) + FVFBlockChunkInfo.FILE_EXTENSION;
var chunkFs = File.OpenRead(Path.Join(blockDir, chunkMd5Name));
foreach (var file in chunk.files)
{
var filePath = Path.Combine(outputDir, file.fileName);
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
Directory.CreateDirectory(Path.GetDirectoryName(filePath) ?? throw new InvalidDataException($"Cannot get directory name of {filePath}"));
if (file.bUseEncrypt)
{
byte[] fileNonce = GC.AllocateUninitializedArray<byte>(VFSDefine.BLOCK_HEAD_LEN);
Buffer.BlockCopy(BitConverter.GetBytes(vfBlockMainInfo.version), 0, fileNonce, 0, sizeof(int));
Buffer.BlockCopy(BitConverter.GetBytes(file.ivSeed), 0, fileNonce, sizeof(int), sizeof(long));
var fileChacha = new CSChaCha20(Convert.FromBase64String(VFSDefine.CHACHA_KEY), fileNonce, 1);
var encryptedMs = new MemoryStream();
chunkFs.CopyBytes(encryptedMs, file.len);
// TODO: Should've used stream decryptor for better perf, but idk how to get it working
File.WriteAllBytes(filePath, fileChacha.DecryptBytes(encryptedMs.ToArray()));
}
else
{
var fileFs = File.OpenWrite(filePath);
chunkFs.Seek(file.offset, SeekOrigin.Begin);
chunkFs.CopyBytes(fileFs, file.len);
fileFs.Dispose();
}
}
Console.WriteLine("Dumped {0} file(s) from chunk {1}", chunk.files.Length, chunkMd5Name);
chunkFs.Dispose();
}
}
}

28
BeyondTools.sln Normal file
View File

@@ -0,0 +1,28 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.12.35527.113
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.VFS", "BeyondTools.VFS\BeyondTools.VFS.csproj", "{673078E8-FBE7-48B3-A228-E7451474625E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeyondTools.SparkBuffer", "BeyondTools.SparkBuffer\BeyondTools.SparkBuffer.csproj", "{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{673078E8-FBE7-48B3-A228-E7451474625E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{673078E8-FBE7-48B3-A228-E7451474625E}.Release|Any CPU.Build.0 = Release|Any CPU
{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{245CFFEA-322D-44D8-BFDA-6340A84D2A9E}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

19
README.md
View File

@@ -1,2 +1,17 @@
# BeyondTools ## BeyondTools.VFS
Dumps VFS assets
```
Usage: [arguments...] [-h|--help] [--version]
Arguments:
[0] <string> game StreamingAssets path (Beyond_Data/StreamingAssets)
[1] <All|Audio|Bundle|BundleManifest|IFixPatch|InitialAudio|InitialBundle|IV|LowShader|Raw|Streaming|TextAsset|Video> dumped asset type
[2] <string> output directory
```
## BeyondTools.SparkBuffer
Dumps SparkBuffer binaries exported from VFS TextAsset
```
Usage: [arguments...] [-h|--help] [--version]
Arguments:
[0] <string> TextAsset/TableCfg directory
[1] <string> output directory
```