- [Core] Added Shader decompilation/disassembly for DXCB program.

AssetStudio.Utility/ShaderConverter.cs

@@ -1,11 +1,15 @@
-using SpirV;
+using AssetStudio.PInvoke;
+using SharpGen.Runtime;
+using SpirV;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
+using Vortice.D3DCompiler;
 
 namespace AssetStudio
 {
@@ -1038,9 +1042,17 @@ namespace AssetStudio
                     case ShaderGpuProgramType.DX9PixelSM20:
                     case ShaderGpuProgramType.DX9PixelSM30:
                         {
-                            /*var shaderBytecode = new ShaderBytecode(m_ProgramCode);
-                            sb.Append(shaderBytecode.Disassemble());*/
-                            sb.Append("// shader disassembly not supported on DXBC");
+                            try
+                            {
+                                var programCodeSpan = m_ProgramCode.AsSpan();
+                                var g = Compiler.Disassemble(programCodeSpan.GetPinnableReference(), programCodeSpan.Length, DisasmFlags.None, "");
+                                sb.Append(g.AsString());
+                            }
+                            catch (Exception e)
+                            {
+                                sb.Append($"// disassembly error {e.Message}\n");
+                            }
+
                             break;
                         }
                     case ShaderGpuProgramType.DX10Level9Vertex:
@@ -1054,16 +1066,42 @@ namespace AssetStudio
                     case ShaderGpuProgramType.DX11HullSM50:
                     case ShaderGpuProgramType.DX11DomainSM50:
                         {
-                            /*int start = 6;
-                            if (m_Version == 201509030) // 5.3
+                            int type = m_ProgramCode[0];
+                            int start = 1;
+                            if (type > 0)
                             {
-                                start = 5;
+                                if (type == 1)
+                                {
+                                    start = 6;
+                                }
+                                else if (type == 2)
+                                {
+                                    start = 38;
+                                }
+                            }
+
+                            var buffSpan = m_ProgramCode.AsSpan(start);
+
+                            try
+                            {
+                                HLSLDecompiler.DecompileShader(buffSpan.ToArray(), buffSpan.Length, out var hlslText);
+                                sb.Append(hlslText);
+                            }
+                            catch (Exception e)
+                            {
+                                Logger.Verbose($"Decompile error {e.Message}");
+                                Logger.Verbose($"Attempting to disassemble...");
+
+                                try
+                                {
+                                    var g = Compiler.Disassemble(buffSpan.GetPinnableReference(), buffSpan.Length, DisasmFlags.None, "");
+                                    sb.Append(g.AsString());
+                                }
+                                catch (Exception ex)
+                                {
+                                    sb.Append($"// decompile/disassembly error {ex.Message}\n");
+                                }
                             }
-                            var buff = new byte[m_ProgramCode.Length - start];
-                            Buffer.BlockCopy(m_ProgramCode, start, buff, 0, buff.Length);
-                            var shaderBytecode = new ShaderBytecode(buff);
-                            sb.Append(shaderBytecode.Disassemble());*/
-                            sb.Append("// shader disassembly not supported on DXBC");
                             break;
                         }
                     case ShaderGpuProgramType.MetalVS:
@@ -1107,4 +1145,31 @@ namespace AssetStudio
             return sb.ToString();
         }
     }
+
+    public static class HLSLDecompiler
+    {
+        private const string DLL_NAME = "HLSLDecompiler";
+        static HLSLDecompiler()
+        {
+            DllLoader.PreloadDll(DLL_NAME);
+        }
+        public static void DecompileShader(byte[] shaderByteCode, int shaderByteCodeSize, out string hlslText)
+        {
+            var code = Decompile(shaderByteCode, shaderByteCodeSize, out var shaderText, out var shaderTextSize);
+            if (code != 0)
+            {
+                throw new Exception($"Unable to decompile shader, Error code: {code}");
+            }
+
+            hlslText = Marshal.PtrToStringAnsi(shaderText, shaderTextSize);
+            Marshal.FreeHGlobal(shaderText);
+        }
+
+        #region importfunctions
+
+        [DllImport(DLL_NAME, CallingConvention = CallingConvention.Cdecl)]
+        private static extern int Decompile(byte[] shaderByteCode, int shaderByteCodeSize, out IntPtr shaderText, out int shaderTextSize);
+
+        #endregion
+    }
 }