Struct reading and disassembly script overhaul, various misc. loading fixes, bump to .NET 9 (#13)

* Bump projects to .net 9 and update nugets

* add VersionedSerialization + source generator

* migrate versioning to StructVersion class, add handling/detection for 29.2/31.2

* add new struct definitions

* rename serialization methods and add BinaryObjectStreamReader for interop

* Rework metadata struct loading to use new struct versioning

* move 29/31.1/.2 to use tags (-2022,-2023) instead of minor versions

* fix metadata usage validity checks

* rework code registration offsetting a bit and add second 29/31.1 condition

* tweak .1 condition (again)

* 29/31.2 was a psyop

* also remove 29.2 from the readme

* remove loading of packed dlls - this was a very unsafe feature

* support auto-recovering type indices from type handles
fixes loading of memory-dumped v29+ libraries since those replacee their class indices on load with a pointer to the corresponding type

* support loading PEs without an export table

* also read UnresolvedVirtualCallCount on regular v31

* Disable plugin loading for now

* Overhaul disassembler script + add Binary Ninja target (#12)

* Overhaul diassembler scripts:
- No longer defines top level functions
- Split into three classes: StatusHandler (like before), DisassemblerInterface (for interfacing with the used program API), ScriptContext (for definiting general functions that use the disassembler interface)
- Add type annotations to all class methods and remove 2.7 compatibility stuff (Ghidra now supports Python 3 so this is unnecessary anymore)
- Disassembler backends are now responsible for launching metadata/script processing, to better support disassembler differences
- String handling is back in the base ScriptContext class, disassembler interfaces opt into the fake string segment creation and fall back to the old method if it isn't supported

* Add Binary Ninja disassembler script backend
This uses the new backend-controlled execution to launch metadata processing on a background thread to keep the ui responsive

* make binary ninja script use own _BINARYNINJA_ define and add define helpers to header

* Update README to account for new script and binary ninja backend

* implement fake string segment functions for binary ninja but don't advertise support

* also cache API function types in binary ninja backend

* fix ida script and disable folders again

* Fix metadata usage issues caused by it being a value type now

* make TryMapVATR overrideable and implement it for ELFs

* Make field offset reading use TryMapVATR to reduce exceptions

* Fix NRE in Assembly ctor on < v24.2

* Update actions workflow to produce cross-platform CLI binaries, update readme to reflect .net 9 changes

* workflow: only restore packages for projects that are being built

* workflow: tweak caching and fix gui compilation

* workflow: remove double .zip in CLI artifact name

* 29/31.2 don't actually exist, this logic is not needed

VersionedSerialization/Attributes/CustomSerializationAttribute.cs

@@ -0,0 +1,6 @@
+namespace VersionedSerialization.Attributes;
+
+[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
+#pragma warning disable CS9113 // Parameter is unread.
+public class CustomSerializationAttribute(string methodName, string sizeExpression) : Attribute;
+#pragma warning restore CS9113 // Parameter is unread.

VersionedSerialization/Attributes/NativeIntegerAttribute.cs

@@ -0,0 +1,3 @@
+namespace VersionedSerialization.Attributes;
+
+public class NativeIntegerAttribute : Attribute;

VersionedSerialization/Attributes/VersionConditionAttribute.cs

@@ -0,0 +1,11 @@
+namespace VersionedSerialization.Attributes;
+
+[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = true)]
+public class VersionConditionAttribute : Attribute
+{
+    public string LessThan { get; set; } = "";
+    public string GreaterThan { get; set; } = "";
+    public string EqualTo { get; set; } = "";
+    public string IncludingTag { get; set; } = "";
+    public string ExcludingTag { get; set; } = "";
+}

VersionedSerialization/Attributes/VersionedStructAttribute.cs

@@ -0,0 +1,4 @@
+namespace VersionedSerialization.Attributes;
+
+[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct)]
+public class VersionedStructAttribute : Attribute;

VersionedSerialization/IReadable.cs

@@ -0,0 +1,7 @@
+namespace VersionedSerialization;
+
+public interface IReadable
+{
+    public void Read<TReader>(ref TReader reader, in StructVersion version = default) where TReader : IReader, allows ref struct;
+    public static abstract int Size(in StructVersion version = default, bool is32Bit = false);
+}

VersionedSerialization/IReader.cs

@@ -0,0 +1,23 @@
+using System.Collections.Immutable;
+
+namespace VersionedSerialization;
+
+public interface IReader
+{
+    bool Is32Bit { get; }
+
+    bool ReadBoolean();
+    long ReadNInt();
+    ulong ReadNUInt();
+    string ReadString();
+    ReadOnlySpan<byte> ReadBytes(int length);
+
+    T ReadPrimitive<T>() where T : unmanaged;
+    ImmutableArray<T> ReadPrimitiveArray<T>(long count) where T : unmanaged;
+
+    T ReadVersionedObject<T>(in StructVersion version = default) where T : IReadable, new();
+    ImmutableArray<T> ReadVersionedObjectArray<T>(long count, in StructVersion version = default) where T : IReadable, new();
+
+    void Align(int alignment = 0);
+    void Skip(int count);
+}

VersionedSerialization/ReaderExtensions.cs

@@ -0,0 +1,62 @@
+using System.Runtime.CompilerServices;
+
+namespace VersionedSerialization;
+
+public static class ReaderExtensions
+{
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static uint ReadCompressedUInt<T>(this ref T reader) where T : struct, IReader, allows ref struct
+    {
+        var first = reader.ReadPrimitive<byte>();
+
+        if ((first & 0b10000000) == 0b00000000)
+            return first;
+
+        if ((first & 0b11000000) == 0b10000000)
+            return (uint)(((first & ~0b10000000) << 8) | reader.ReadPrimitive<byte>());
+
+        if ((first & 0b11100000) == 0b11000000)
+            return (uint)(((first & ~0b11000000) << 24) | (reader.ReadPrimitive<byte>() << 16) | (reader.ReadPrimitive<byte>() << 8) | reader.ReadPrimitive<byte>());
+
+        return first switch
+        {
+            0b11110000 => reader.ReadPrimitive<uint>(),
+            0b11111110 => uint.MaxValue - 1,
+            0b11111111 => uint.MaxValue,
+            _ => throw new InvalidDataException("Invalid compressed uint")
+        };
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int ReadCompressedInt<T>(this ref T reader) where T : struct, IReader, allows ref struct
+    {
+        var value = reader.ReadCompressedUInt();
+        if (value == uint.MaxValue)
+            return int.MinValue;
+
+        var isNegative = (value & 0b1) == 1;
+        value >>= 1;
+
+        return (int)(isNegative ? -(value + 1) : value);
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static ulong ReadSLEB128<T>(this ref T reader) where T : struct, IReader, allows ref struct
+    {
+        var value = 0uL;
+        var shift = 0;
+        byte current;
+
+        do
+        {
+            current = reader.ReadPrimitive<byte>();
+            value |= (current & 0x7FuL) << shift;
+            shift += 7;
+        } while ((current & 0x80) != 0);
+
+        if (64 >= shift && (current & 0x40) != 0)
+            value |= ulong.MaxValue << shift;
+
+        return value;
+    }
+}

VersionedSerialization/SpanReader.cs

@@ -0,0 +1,151 @@
+using System.Buffers.Binary;
+using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace VersionedSerialization;
+
+// ReSharper disable ReplaceSliceWithRangeIndexer | The range indexer gets compiled into .Slice(x, y) and not .Slice(x) which worsens performance
+public ref struct SpanReader(ReadOnlySpan<byte> data, int offset = 0, bool littleEndian = true, bool is32Bit = false) : IReader
+{
+    public int Offset = offset;
+    public readonly byte Peek => _data[Offset];
+    public readonly bool IsLittleEndian => _littleEndian;
+    public readonly bool Is32Bit => _is32Bit;
+    public readonly int Length => _data.Length;
+    public readonly int PointerSize => Is32Bit ? sizeof(uint) : sizeof(ulong);
+
+    private readonly ReadOnlySpan<byte> _data = data;
+    private readonly bool _littleEndian = littleEndian;
+    private readonly bool _is32Bit = is32Bit;
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private T ReadInternal<T>() where T : unmanaged
+    {
+        var value = MemoryMarshal.Read<T>(_data.Slice(Offset));
+        Offset += Unsafe.SizeOf<T>();
+        return value;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static TTo Cast<TFrom, TTo>(in TFrom from) => Unsafe.As<TFrom, TTo>(ref Unsafe.AsRef(in from));
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public ReadOnlySpan<byte> ReadBytes(int length)
+    {
+        var val = _data.Slice(Offset, length);
+        Offset += length;
+        return val;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public T ReadPrimitive<T>() where T : unmanaged
+    {
+        if (typeof(T) == typeof(byte))
+            return Cast<byte, T>(_data[Offset++]);
+
+        var value = ReadInternal<T>();
+        if (!_littleEndian)
+        {
+            if (value is ulong val)
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(val);
+                value = Cast<ulong, T>(converted);
+            } 
+            else if (typeof(T) == typeof(long))
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(Cast<T, long>(value));
+                value = Cast<long, T>(converted);
+            }
+            else if (typeof(T) == typeof(uint))
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(Cast<T, uint>(value));
+                value = Cast<uint, T>(converted);
+            }
+            else if (typeof(T) == typeof(int))
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(Cast<T, int>(value));
+                value = Cast<int, T>(converted);
+            }
+            else if (typeof(T) == typeof(ushort))
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(Cast<T, ushort>(value));
+                value = Cast<ushort, T>(converted);
+            }
+            else if (typeof(T) == typeof(short))
+            {
+                var converted = BinaryPrimitives.ReverseEndianness(Cast<T, short>(value));
+                value = Cast<short, T>(converted);
+            }
+        }
+
+        return value;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public ImmutableArray<T> ReadPrimitiveArray<T>(long count) where T : unmanaged
+    {
+        var array = ImmutableArray.CreateBuilder<T>(checked((int)count));
+        for (long i = 0; i < count; i++)
+            array.Add(ReadPrimitive<T>());
+
+        return array.MoveToImmutable();
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public T ReadVersionedObject<T>(in StructVersion version = default) where T : IReadable, new()
+    {
+        var obj = new T();
+        obj.Read(ref this, in version);
+        return obj;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public ImmutableArray<T> ReadVersionedObjectArray<T>(long count, in StructVersion version = default) where T : IReadable, new() 
+    {
+        var array = ImmutableArray.CreateBuilder<T>(checked((int)count));
+        for (long i = 0; i < count; i++)
+            array.Add(ReadVersionedObject<T>(in version));
+
+        return array.MoveToImmutable();
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public string ReadString()
+    {
+        var length = _data.Slice(Offset).IndexOf(byte.MinValue);
+
+        if (length == -1)
+            throw new InvalidDataException("Failed to find string in span.");
+
+        var val = _data.Slice(Offset, length);
+        Offset += length + 1; // Skip null terminator
+
+        return Encoding.UTF8.GetString(val);
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool ReadBoolean() => ReadPrimitive<byte>() != 0;
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public ulong ReadNUInt() => _is32Bit ? ReadPrimitive<uint>() : ReadPrimitive<ulong>();
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public long ReadNInt() => _is32Bit ? ReadPrimitive<int>() : ReadPrimitive<long>();
+
+    public void Align(int alignment = 0)
+    {
+        if (alignment == 0)
+            alignment = Is32Bit ? 4 : 8;
+
+        var rem = Offset % alignment;
+        if (rem != 0)
+            Offset += alignment - rem;
+    }
+
+    public void Skip(int count)
+    {
+        Offset += count;
+    }
+}

VersionedSerialization/StructVersion.cs

@@ -0,0 +1,68 @@
+namespace VersionedSerialization;
+
+public readonly struct StructVersion(int major = 0, int minor = 0, string? tag = null) : IEquatable<StructVersion>
+{
+    public readonly int Major = major;
+    public readonly int Minor = minor;
+    public readonly string? Tag = tag;
+
+    public double AsDouble => Major + Minor / 10.0;
+
+    #region Equality operators
+
+    public static bool operator ==(StructVersion left, StructVersion right)
+        => left.Major == right.Major && left.Minor == right.Minor;
+
+    public static bool operator !=(StructVersion left, StructVersion right)
+        => !(left == right);
+
+    public static bool operator >(StructVersion left, StructVersion right)
+        => left.Major > right.Major || (left.Major == right.Major && left.Minor > right.Minor);
+
+    public static bool operator <(StructVersion left, StructVersion right)
+        => left.Major < right.Major || (left.Major == right.Major && left.Minor < right.Minor);
+
+    public static bool operator >=(StructVersion left, StructVersion right)
+        => left.Major > right.Major || (left.Major == right.Major && left.Minor >= right.Minor);
+
+    public static bool operator <=(StructVersion left, StructVersion right)
+        => left.Major < right.Major || (left.Major == right.Major && left.Minor <= right.Minor);
+
+    public override bool Equals(object? obj)
+        => obj is StructVersion other && Equals(other);
+
+    public bool Equals(StructVersion other)
+        => Major == other.Major && Minor == other.Minor;
+
+    public override int GetHashCode()
+        => HashCode.Combine(Major, Minor);
+
+    #endregion
+
+    public override string ToString() => $"{Major}.{Minor}{(Tag != null ? $"-{Tag}" : "")}";
+
+    public static implicit operator StructVersion(string value)
+    {
+        var versionParts = value.Split('.');
+        if (versionParts.Length > 2)
+            throw new InvalidOperationException("Invalid version string.");
+
+        if (versionParts.Length == 1)
+        {
+            if (!int.TryParse(versionParts[0], out var version))
+                throw new InvalidOperationException("Invalid single-number version string.");
+
+            return new StructVersion(version);
+        }
+
+        var tagParts = versionParts[1].Split("-");
+        if (tagParts.Length > 2) 
+            throw new InvalidOperationException("Invalid version string.");
+
+        var major = int.Parse(versionParts[0]);
+        var minor = int.Parse(tagParts[0]);
+        var tag = tagParts.Length == 1 ? null : tagParts[1];
+
+        return new StructVersion(major, minor, tag);
+    }
+}

VersionedSerialization/VersionedSerialization.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+	<LangVersion>preview</LangVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <IsAotCompatible>True</IsAotCompatible>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <IsAotCompatible>True</IsAotCompatible>
+  </PropertyGroup>
+
+</Project>