Struct reading and disassembly script overhaul, various misc. loading fixes, bump to .NET 9 (#13)

* Bump projects to .net 9 and update nugets

* add VersionedSerialization + source generator

* migrate versioning to StructVersion class, add handling/detection for 29.2/31.2

* add new struct definitions

* rename serialization methods and add BinaryObjectStreamReader for interop

* Rework metadata struct loading to use new struct versioning

* move 29/31.1/.2 to use tags (-2022,-2023) instead of minor versions

* fix metadata usage validity checks

* rework code registration offsetting a bit and add second 29/31.1 condition

* tweak .1 condition (again)

* 29/31.2 was a psyop

* also remove 29.2 from the readme

* remove loading of packed dlls - this was a very unsafe feature

* support auto-recovering type indices from type handles
fixes loading of memory-dumped v29+ libraries since those replacee their class indices on load with a pointer to the corresponding type

* support loading PEs without an export table

* also read UnresolvedVirtualCallCount on regular v31

* Disable plugin loading for now

* Overhaul disassembler script + add Binary Ninja target (#12)

* Overhaul diassembler scripts:
- No longer defines top level functions
- Split into three classes: StatusHandler (like before), DisassemblerInterface (for interfacing with the used program API), ScriptContext (for definiting general functions that use the disassembler interface)
- Add type annotations to all class methods and remove 2.7 compatibility stuff (Ghidra now supports Python 3 so this is unnecessary anymore)
- Disassembler backends are now responsible for launching metadata/script processing, to better support disassembler differences
- String handling is back in the base ScriptContext class, disassembler interfaces opt into the fake string segment creation and fall back to the old method if it isn't supported

* Add Binary Ninja disassembler script backend
This uses the new backend-controlled execution to launch metadata processing on a background thread to keep the ui responsive

* make binary ninja script use own _BINARYNINJA_ define and add define helpers to header

* Update README to account for new script and binary ninja backend

* implement fake string segment functions for binary ninja but don't advertise support

* also cache API function types in binary ninja backend

* fix ida script and disable folders again

* Fix metadata usage issues caused by it being a value type now

* make TryMapVATR overrideable and implement it for ELFs

* Make field offset reading use TryMapVATR to reduce exceptions

* Fix NRE in Assembly ctor on < v24.2

* Update actions workflow to produce cross-platform CLI binaries, update readme to reflect .net 9 changes

* workflow: only restore packages for projects that are being built

* workflow: tweak caching and fix gui compilation

* workflow: remove double .zip in CLI artifact name

* 29/31.2 don't actually exist, this logic is not needed
This commit is contained in:
Luke
2024-11-14 14:32:11 +01:00
committed by GitHub
parent 5b0476fcc5
commit b05c03964a
130 changed files with 5117 additions and 4371 deletions

View File

@@ -7,41 +7,45 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reflection;
using Il2CppInspector.Next;
using Il2CppInspector.Next.Metadata;
using NoisyCowStudios.Bin2Object;
using VersionedSerialization;
namespace Il2CppInspector
{
public class Metadata : BinaryObjectStream
public class Metadata : BinaryObjectStreamReader
{
public Il2CppGlobalMetadataHeader Header { get; set; }
public Il2CppAssemblyDefinition[] Assemblies { get; set; }
public Il2CppImageDefinition[] Images { get; set; }
public Il2CppTypeDefinition[] Types { get; set; }
public Il2CppMethodDefinition[] Methods { get; set; }
public Il2CppParameterDefinition[] Params { get; set; }
public Il2CppFieldDefinition[] Fields { get; set; }
public Il2CppFieldDefaultValue[] FieldDefaultValues { get; set; }
public Il2CppParameterDefaultValue[] ParameterDefaultValues { get; set; }
public Il2CppPropertyDefinition[] Properties { get; set; }
public Il2CppEventDefinition[] Events { get; set; }
public Il2CppGenericContainer[] GenericContainers { get; set; }
public Il2CppGenericParameter[] GenericParameters { get; set; }
public Il2CppCustomAttributeTypeRange[] AttributeTypeRanges { get; set; }
public Il2CppCustomAttributeDataRange[] AttributeDataRanges { get; set; }
public Il2CppInterfaceOffsetPair[] InterfaceOffsets { get; set; }
public Il2CppMetadataUsageList[] MetadataUsageLists { get; set; }
public Il2CppMetadataUsagePair[] MetadataUsagePairs { get; set; }
public Il2CppFieldRef[] FieldRefs { get; set; }
public ImmutableArray<Il2CppAssemblyDefinition> Assemblies { get; set; }
public ImmutableArray<Il2CppImageDefinition> Images { get; set; }
public ImmutableArray<Il2CppTypeDefinition> Types { get; set; }
public ImmutableArray<Il2CppMethodDefinition> Methods { get; set; }
public ImmutableArray<Il2CppParameterDefinition> Params { get; set; }
public ImmutableArray<Il2CppFieldDefinition> Fields { get; set; }
public ImmutableArray<Il2CppFieldDefaultValue> FieldDefaultValues { get; set; }
public ImmutableArray<Il2CppParameterDefaultValue> ParameterDefaultValues { get; set; }
public ImmutableArray<Il2CppPropertyDefinition> Properties { get; set; }
public ImmutableArray<Il2CppEventDefinition> Events { get; set; }
public ImmutableArray<Il2CppGenericContainer> GenericContainers { get; set; }
public ImmutableArray<Il2CppGenericParameter> GenericParameters { get; set; }
public ImmutableArray<Il2CppCustomAttributeTypeRange> AttributeTypeRanges { get; set; }
public ImmutableArray<Il2CppCustomAttributeDataRange> AttributeDataRanges { get; set; }
public ImmutableArray<Il2CppInterfaceOffsetPair> InterfaceOffsets { get; set; }
public ImmutableArray<Il2CppMetadataUsageList> MetadataUsageLists { get; set; }
public ImmutableArray<Il2CppMetadataUsagePair> MetadataUsagePairs { get; set; }
public ImmutableArray<Il2CppFieldRef> FieldRefs { get; set; }
public int[] InterfaceUsageIndices { get; set; }
public int[] NestedTypeIndices { get; set; }
public int[] AttributeTypeIndices { get; set; }
public int[] GenericConstraintIndices { get; set; }
public uint[] VTableMethodIndices { get; set; }
public ImmutableArray<int> InterfaceUsageIndices { get; set; }
public ImmutableArray<int> NestedTypeIndices { get; set; }
public ImmutableArray<int> AttributeTypeIndices { get; set; }
public ImmutableArray<int> GenericConstraintIndices { get; set; }
public ImmutableArray<uint> VTableMethodIndices { get; set; }
public string[] StringLiterals { get; set; }
public Dictionary<int, string> Strings { get; private set; } = new Dictionary<int, string>();
@@ -78,22 +82,22 @@ namespace Il2CppInspector
StatusUpdate("Processing metadata");
// Read metadata header
Header = ReadObject<Il2CppGlobalMetadataHeader>(0);
Header = ReadVersionedObject<Il2CppGlobalMetadataHeader>(0);
// Check for correct magic bytes
if (Header.signature != Il2CppConstants.MetadataSignature) {
if (!Header.SanityValid) {
throw new InvalidOperationException("The supplied metadata file is not valid.");
}
// Set object versioning for Bin2Object from metadata version
Version = Header.version;
Version = new StructVersion(Header.Version);
if (Version < 16 || Version > 31) {
throw new InvalidOperationException($"The supplied metadata file is not of a supported version ({Header.version}).");
if (Version < MetadataVersions.V160 || Version > MetadataVersions.V310) {
throw new InvalidOperationException($"The supplied metadata file is not of a supported version ({Header.Version}).");
}
// Rewind and read metadata header with the correct version settings
Header = ReadObject<Il2CppGlobalMetadataHeader>(0);
Header = ReadVersionedObject<Il2CppGlobalMetadataHeader>(0);
// Sanity checking
// Unity.IL2CPP.MetadataCacheWriter.WriteLibIl2CppMetadata always writes the metadata information in the same order it appears in the header,
@@ -105,109 +109,90 @@ namespace Il2CppInspector
// we can use this value to determine the actual header length and therefore narrow down the metadata version to 24.0/24.1 or 24.2.
if (!pluginResult.SkipValidation) {
var realHeaderLength = Header.stringLiteralOffset;
var realHeaderLength = Header.StringLiteralOffset;
if (realHeaderLength != Sizeof(typeof(Il2CppGlobalMetadataHeader))) {
if (Version == 24.0) {
Version = 24.2;
Header = ReadObject<Il2CppGlobalMetadataHeader>(0);
if (realHeaderLength != Sizeof<Il2CppGlobalMetadataHeader>()) {
if (Version == MetadataVersions.V240) {
Version = MetadataVersions.V242;
Header = ReadVersionedObject<Il2CppGlobalMetadataHeader>(0);
}
}
if (realHeaderLength != Sizeof(typeof(Il2CppGlobalMetadataHeader))) {
if (realHeaderLength != Sizeof<Il2CppGlobalMetadataHeader>()) {
throw new InvalidOperationException("Could not verify the integrity of the metadata file or accurately identify the metadata sub-version");
}
}
// Load all the relevant metadata using offsets provided in the header
if (Version >= 16)
Images = ReadArray<Il2CppImageDefinition>(Header.imagesOffset, Header.imagesCount / Sizeof(typeof(Il2CppImageDefinition)));
if (Version >= MetadataVersions.V160)
Images = ReadVersionedObjectArray<Il2CppImageDefinition>(Header.ImagesOffset, Header.ImagesSize / Sizeof<Il2CppImageDefinition>());
// As an additional sanity check, all images in the metadata should have Mono.Cecil.MetadataToken == 1
// In metadata v24.1, two extra fields were added which will cause the below test to fail.
// In that case, we can then adjust the version number and reload
// Tokens were introduced in v19 - we don't bother testing earlier versions
if (Version >= 19 && Images.Any(x => x.token != 1))
if (Version == 24.0) {
Version = 24.1;
if (Version >= MetadataVersions.V190 && Images.Any(x => x.Token != 1))
if (Version == MetadataVersions.V240) {
Version = MetadataVersions.V241;
// No need to re-read the header, it's the same for both sub-versions
Images = ReadArray<Il2CppImageDefinition>(Header.imagesOffset, Header.imagesCount / Sizeof(typeof(Il2CppImageDefinition)));
Images = ReadVersionedObjectArray<Il2CppImageDefinition>(Header.ImagesOffset, Header.ImagesSize / Sizeof<Il2CppImageDefinition>());
if (Images.Any(x => x.token != 1))
if (Images.Any(x => x.Token != 1))
throw new InvalidOperationException("Could not verify the integrity of the metadata file image list");
}
Types = ReadArray<Il2CppTypeDefinition>(Header.typeDefinitionsOffset, Header.typeDefinitionsCount / Sizeof(typeof(Il2CppTypeDefinition)));
Methods = ReadArray<Il2CppMethodDefinition>(Header.methodsOffset, Header.methodsCount / Sizeof(typeof(Il2CppMethodDefinition)));
Params = ReadArray<Il2CppParameterDefinition>(Header.parametersOffset, Header.parametersCount / Sizeof(typeof(Il2CppParameterDefinition)));
Fields = ReadArray<Il2CppFieldDefinition>(Header.fieldsOffset, Header.fieldsCount / Sizeof(typeof(Il2CppFieldDefinition)));
FieldDefaultValues = ReadArray<Il2CppFieldDefaultValue>(Header.fieldDefaultValuesOffset, Header.fieldDefaultValuesCount / Sizeof(typeof(Il2CppFieldDefaultValue)));
Properties = ReadArray<Il2CppPropertyDefinition>(Header.propertiesOffset, Header.propertiesCount / Sizeof(typeof(Il2CppPropertyDefinition)));
Events = ReadArray<Il2CppEventDefinition>(Header.eventsOffset, Header.eventsCount / Sizeof(typeof(Il2CppEventDefinition)));
InterfaceUsageIndices = ReadArray<int>(Header.interfacesOffset, Header.interfacesCount / sizeof(int));
NestedTypeIndices = ReadArray<int>(Header.nestedTypesOffset, Header.nestedTypesCount / sizeof(int));
GenericContainers = ReadArray<Il2CppGenericContainer>(Header.genericContainersOffset, Header.genericContainersCount / Sizeof(typeof(Il2CppGenericContainer)));
GenericParameters = ReadArray<Il2CppGenericParameter>(Header.genericParametersOffset, Header.genericParametersCount / Sizeof(typeof(Il2CppGenericParameter)));
GenericConstraintIndices = ReadArray<int>(Header.genericParameterConstraintsOffset, Header.genericParameterConstraintsCount / sizeof(int));
InterfaceOffsets = ReadArray<Il2CppInterfaceOffsetPair>(Header.interfaceOffsetsOffset, Header.interfaceOffsetsCount / Sizeof(typeof(Il2CppInterfaceOffsetPair)));
VTableMethodIndices = ReadArray<uint>(Header.vtableMethodsOffset, Header.vtableMethodsCount / sizeof(uint));
Types = ReadVersionedObjectArray<Il2CppTypeDefinition>(Header.TypeDefinitionsOffset, Header.TypeDefinitionsSize / Sizeof<Il2CppTypeDefinition>());
Methods = ReadVersionedObjectArray<Il2CppMethodDefinition>(Header.MethodsOffset, Header.MethodsSize / Sizeof<Il2CppMethodDefinition>());
Params = ReadVersionedObjectArray<Il2CppParameterDefinition>(Header.ParametersOffset, Header.ParametersSize / Sizeof<Il2CppParameterDefinition>());
Fields = ReadVersionedObjectArray<Il2CppFieldDefinition>(Header.FieldsOffset, Header.FieldsSize / Sizeof<Il2CppFieldDefinition>());
FieldDefaultValues = ReadVersionedObjectArray<Il2CppFieldDefaultValue>(Header.FieldDefaultValuesOffset, Header.FieldDefaultValuesSize / Sizeof<Il2CppFieldDefaultValue>());
Properties = ReadVersionedObjectArray<Il2CppPropertyDefinition>(Header.PropertiesOffset, Header.PropertiesSize / Sizeof<Il2CppPropertyDefinition>());
Events = ReadVersionedObjectArray<Il2CppEventDefinition>(Header.EventsOffset, Header.EventsSize / Sizeof<Il2CppEventDefinition>());
InterfaceUsageIndices = ReadPrimitiveArray<int>(Header.InterfacesOffset, Header.InterfacesSize / sizeof(int));
NestedTypeIndices = ReadPrimitiveArray<int>(Header.NestedTypesOffset, Header.NestedTypesSize / sizeof(int));
GenericContainers = ReadVersionedObjectArray<Il2CppGenericContainer>(Header.GenericContainersOffset, Header.GenericContainersSize / Sizeof<Il2CppGenericContainer>());
GenericParameters = ReadVersionedObjectArray<Il2CppGenericParameter>(Header.GenericParametersOffset, Header.GenericParametersSize / Sizeof<Il2CppGenericParameter>());
GenericConstraintIndices = ReadPrimitiveArray<int>(Header.GenericParameterConstraintsOffset, Header.GenericParameterConstraintsSize / sizeof(int));
InterfaceOffsets = ReadVersionedObjectArray<Il2CppInterfaceOffsetPair>(Header.InterfaceOffsetsOffset, Header.InterfaceOffsetsSize / Sizeof<Il2CppInterfaceOffsetPair>());
VTableMethodIndices = ReadPrimitiveArray<uint>(Header.VTableMethodsOffset, Header.VTableMethodsSize / sizeof(uint));
if (Version >= 16) {
if (Version >= MetadataVersions.V160) {
// In v24.4 hashValueIndex was removed from Il2CppAssemblyNameDefinition, which is a field in Il2CppAssemblyDefinition
// The number of images and assemblies should be the same. If they are not, we deduce that we are using v24.4
// Note the version comparison matches both 24.2 and 24.3 here since 24.3 is tested for during binary loading
var assemblyCount = Header.assembliesCount / Sizeof(typeof(Il2CppAssemblyDefinition));
var assemblyCount = Header.AssembliesSize / Sizeof<Il2CppAssemblyDefinition>();
var changedAssemblyDefStruct = false;
if ((Version == 24.1 || Version == 24.2 || Version == 24.3) && assemblyCount < Images.Length)
if ((Version == MetadataVersions.V241 || Version == MetadataVersions.V242 || Version == MetadataVersions.V243) && assemblyCount < Images.Length)
{
if (Version == 24.1)
if (Version == MetadataVersions.V241)
changedAssemblyDefStruct = true;
Version = 24.4;
Version = MetadataVersions.V244;
}
Assemblies = ReadArray<Il2CppAssemblyDefinition>(Header.assembliesOffset, Images.Length);
Assemblies = ReadVersionedObjectArray<Il2CppAssemblyDefinition>(Header.AssembliesOffset, Images.Length);
if (changedAssemblyDefStruct)
Version = 24.1;
Version = MetadataVersions.V241;
ParameterDefaultValues = ReadArray<Il2CppParameterDefaultValue>(Header.parameterDefaultValuesOffset, Header.parameterDefaultValuesCount / Sizeof(typeof(Il2CppParameterDefaultValue)));
ParameterDefaultValues = ReadVersionedObjectArray<Il2CppParameterDefaultValue>(Header.ParameterDefaultValuesOffset, Header.ParameterDefaultValuesSize / Sizeof<Il2CppParameterDefaultValue>());
}
if (Version >= 19 && Version < 27) {
MetadataUsageLists = ReadArray<Il2CppMetadataUsageList>(Header.metadataUsageListsOffset, Header.metadataUsageListsCount / Sizeof(typeof(Il2CppMetadataUsageList)));
MetadataUsagePairs = ReadArray<Il2CppMetadataUsagePair>(Header.metadataUsagePairsOffset, Header.metadataUsagePairsCount / Sizeof(typeof(Il2CppMetadataUsagePair)));
if (Version >= MetadataVersions.V190 && Version < MetadataVersions.V270) {
MetadataUsageLists = ReadVersionedObjectArray<Il2CppMetadataUsageList>(Header.MetadataUsageListsOffset, Header.MetadataUsageListsCount / Sizeof<Il2CppMetadataUsageList>());
MetadataUsagePairs = ReadVersionedObjectArray<Il2CppMetadataUsagePair>(Header.MetadataUsagePairsOffset, Header.MetadataUsagePairsCount / Sizeof<Il2CppMetadataUsagePair>());
}
if (Version >= 19) {
FieldRefs = ReadArray<Il2CppFieldRef>(Header.fieldRefsOffset, Header.fieldRefsCount / Sizeof(typeof(Il2CppFieldRef)));
if (Version >= MetadataVersions.V190) {
FieldRefs = ReadVersionedObjectArray<Il2CppFieldRef>(Header.FieldRefsOffset, Header.FieldRefsSize / Sizeof<Il2CppFieldRef>());
}
if (Version >= 21 && Version < 29) {
AttributeTypeIndices = ReadArray<int>(Header.attributeTypesOffset, Header.attributeTypesCount / sizeof(int));
AttributeTypeRanges = ReadArray<Il2CppCustomAttributeTypeRange>(Header.attributesInfoOffset, Header.attributesInfoCount / Sizeof(typeof(Il2CppCustomAttributeTypeRange)));
if (Version >= MetadataVersions.V210 && Version < MetadataVersions.V290) {
AttributeTypeIndices = ReadPrimitiveArray<int>(Header.AttributesTypesOffset, Header.AttributesTypesCount / sizeof(int));
AttributeTypeRanges = ReadVersionedObjectArray<Il2CppCustomAttributeTypeRange>(Header.AttributesInfoOffset, Header.AttributesInfoCount / Sizeof<Il2CppCustomAttributeTypeRange>());
}
if (Version >= 29)
if (Version >= MetadataVersions.V290)
{
AttributeDataRanges = ReadArray<Il2CppCustomAttributeDataRange>(Header.attributeDataRangeOffset,
Header.attributeDataRangeSize / Sizeof(typeof(Il2CppCustomAttributeDataRange)));
}
if (Version is 29 or 31)
{
// 29.2/31.2 added a new isUnmanagedCallersOnly flag to Il2CppMethodDefinition.
// This offsets all subsequent entries by one - we can detect this by checking the
// top token byte (which should always be 0x06).
if (Methods.Length >= 2)
{
var secondToken = Methods[1].token;
if (secondToken >> 24 != 0x6)
{
Version += 0.2;
Methods = ReadArray<Il2CppMethodDefinition>(Header.methodsOffset,
Header.methodsCount / Sizeof(typeof(Il2CppMethodDefinition)));
}
}
AttributeDataRanges = ReadVersionedObjectArray<Il2CppCustomAttributeDataRange>(Header.AttributeDataRangeOffset,
Header.AttributeDataRangeSize / Sizeof<Il2CppCustomAttributeDataRange>());
}
// Get all metadata strings
@@ -216,10 +201,10 @@ namespace Il2CppInspector
Strings = pluginGetStringsResult.Strings;
else {
Position = Header.stringOffset;
Position = Header.StringOffset;
while (Position < Header.stringOffset + Header.stringCount)
Strings.Add((int) Position - Header.stringOffset, ReadNullTerminatedString());
while (Position < Header.StringOffset + Header.StringSize)
Strings.Add((int) Position - Header.StringOffset, ReadNullTerminatedString());
}
// Get all string literals
@@ -228,11 +213,11 @@ namespace Il2CppInspector
StringLiterals = pluginGetStringLiteralsResult.StringLiterals.ToArray();
else {
var stringLiteralList = ReadArray<Il2CppStringLiteral>(Header.stringLiteralOffset, Header.stringLiteralCount / Sizeof(typeof(Il2CppStringLiteral)));
var stringLiteralList = ReadVersionedObjectArray<Il2CppStringLiteral>(Header.StringLiteralOffset, Header.StringLiteralSize / Sizeof<Il2CppStringLiteral>());
StringLiterals = new string[stringLiteralList.Length];
for (var i = 0; i < stringLiteralList.Length; i++)
StringLiterals[i] = ReadFixedLengthString(Header.stringLiteralDataOffset + stringLiteralList[i].dataIndex, stringLiteralList[i].length);
StringLiterals[i] = ReadFixedLengthString(Header.StringLiteralDataOffset + stringLiteralList[i].DataIndex, (int)stringLiteralList[i].Length);
}
// Post-processing hook
@@ -246,40 +231,6 @@ namespace Il2CppInspector
CopyTo(outFile);
}
public int Sizeof(Type type) => Sizeof(type, Version);
public int Sizeof(Type type, double metadataVersion, int longSizeBytes = 8) {
if (Reader.ObjectMappings.TryGetValue(type, out var streamType))
type = streamType;
int size = 0;
foreach (var i in type.GetTypeInfo().GetFields())
{
// Only process fields for our selected object versioning (always process if none supplied)
var versions = i.GetCustomAttributes<VersionAttribute>(false).Select(v => (v.Min, v.Max)).ToList();
if (versions.Any() && !versions.Any(v => (v.Min <= metadataVersion || v.Min == -1) && (v.Max >= metadataVersion || v.Max == -1)))
continue;
if (i.FieldType == typeof(long) || i.FieldType == typeof(ulong))
size += longSizeBytes;
else if (i.FieldType == typeof(int) || i.FieldType == typeof(uint))
size += 4;
else if (i.FieldType == typeof(short) || i.FieldType == typeof(ushort))
size += 2;
// Fixed-length array
else if (i.FieldType.IsArray) {
var attr = i.GetCustomAttribute<ArrayLengthAttribute>(false) ??
throw new InvalidOperationException("Array field " + i.Name + " must have ArrayLength attribute");
size += attr.FixedSize;
}
// Embedded object
else
size += Sizeof(i.FieldType, metadataVersion);
}
return size;
}
public int Sizeof<T>() where T : IReadable => T.Size(Version, Is32Bit);
}
}