Add option for sample rate alpha instead of alpha to coverage

TellowKrinkle · TellowKrinkle · commit 5a35de564d71 · 2023-10-21T01:18:55.000-05:00
diff --git a/Environment.hlsl b/Environment.hlsl
@@ -85,7 +85,13 @@ struct PSInput
 #endif
 };
 
-float4 main(float4 pos : SV_Position, PSInput input) : SV_TARGET{
+#if ALPHA == 2 && MSAA_SAMPLE_COUNT != 0
+	#define COVERAGE_OUT , out uint coverage : SV_Coverage
+#else
+	#define COVERAGE_OUT
+#endif
+
+float4 main(float4 pos : SV_Position, PSInput input COVERAGE_OUT) : SV_TARGET {
 #if ALPHA != 0
 	float3 lit = sLit.Sample(smpsLit, input.litCoord.xy).xyz * input.litColor;
 	float aref = vATest;
@@ -103,10 +109,28 @@ float4 main(float4 pos : SV_Position, PSInput input) : SV_TARGET{
 #if ALPHA == 1
 	clip(color.a - aref);
 #elif ALPHA == 2
+#if MSAA_SAMPLE_COUNT == 0
 	float pxAlpha = abs(ddx_fine(color.a)) + abs(ddy_fine(color.a));
 	float a2c = saturate((color.a - aref) / pxAlpha + 0.5);
 	if (a2c == 0)
 		discard;
+#else // MSAA_SAMPLE_COUNT == 0
+	if (ddx_coarse(color.a) != 0 || ddy_coarse(color.a) != 0) {
+		// In-shader AA
+		coverage = 0;
+		[unroll]
+		for (uint bit = 0; bit < MSAA_SAMPLE_COUNT; bit++) {
+			coverage |= (sStage0.Sample(smpsStage0, EvaluateAttributeAtSample(input.litCoord, bit).zw).a > aref) << bit;
+		}
+#if MSAA_SAMPLE_COUNT == 16
+		// 16 is the max we compile for
+		coverage |= coverage << 16;
+#endif
+	} else {
+		clip(color.a - aref);
+		coverage = ~0;
+	}
+#endif // MSAA_SAMPLE_COUNT == 0
 #endif
 #endif
 	float2 shadowBase = (input.shadow.xy / input.shadow.w);
@@ -237,11 +261,10 @@ float4 main(float4 pos : SV_Position, PSInput input) : SV_TARGET{
 	color.a *= input.litBase.a;
 	color.rgb = lerp(input.baseColor, lit * color.rgb, input.texColorWeight);
 	color = color * saturate(atColor) + saturate(atColor - 1);
-#if ALPHA == 1
+#if ALPHA == 1 || MSAA_SAMPLE_COUNT != 0
 	return float4(color.rgb * HdrRangeInv, color.a);
 #else
 	return float4(color.rgb * HdrRangeInv, a2c);
 #endif
-
 #endif
 }
diff --git a/ShaderSampleRateConverter.cpp b/ShaderSampleRateConverter.cpp
@@ -58,8 +58,6 @@ bool shouldUseSampleRate(const void* data, SIZE_T length) {
   for (const ShaderHash& hash : sampleRateShaders)
     if (config.ssaaCharacters && hash == header->hash)
       return true;
-  if (config.ssaaTransparentObjects && header->hash == ShaderHash({ 0x0cd1b9e5, 0x22e7069e, 0x476455ff, 0x98bfd850 }))
-      return true;
   return config.ssaaAll;
 }
 
@@ -136,7 +134,21 @@ Buffer getReplacementShader(const void* data, SIZE_T length) {
 }
 
 Buffer getAlphaToCoverageShader(const void* data, SIZE_T length) {
-  return findReplacement(data, length, shaderReplacementsAlphaToCoverage);
+  ShaderReplacementList list;
+  if (config.msaaSamples < 2) {
+      return {};
+  } else if (!config.sampleRateAlpha) {
+    list = shaderReplacementsAlphaToCoverage0;
+  } else if (config.msaaSamples > 8) {
+    list = shaderReplacementsAlphaToCoverage16;
+  } else if (config.msaaSamples > 4) {
+    list = shaderReplacementsAlphaToCoverage8;
+  } else if (config.msaaSamples > 2) {
+    list = shaderReplacementsAlphaToCoverage4;
+  } else {
+    list = shaderReplacementsAlphaToCoverage2;
+  }
+  return findReplacement(data, length, list);
 }
 
 } // namespace atfix
diff --git a/Shaders.h b/Shaders.h
@@ -27,5 +27,9 @@ struct ShaderReplacementList {
 };
 
 extern const ShaderReplacementList shaderReplacements;
-extern const ShaderReplacementList shaderReplacementsAlphaToCoverage;
+extern const ShaderReplacementList shaderReplacementsAlphaToCoverage0;
+extern const ShaderReplacementList shaderReplacementsAlphaToCoverage2;
+extern const ShaderReplacementList shaderReplacementsAlphaToCoverage4;
+extern const ShaderReplacementList shaderReplacementsAlphaToCoverage8;
+extern const ShaderReplacementList shaderReplacementsAlphaToCoverage16;
 }
diff --git a/build-shaders.ps1 b/build-shaders.ps1
@@ -5,7 +5,13 @@ $header += "namespace atfix {`n"
 Set-Content -NoNewline -Path Shaders.cpp $header
 
 $shaders = [System.Collections.ArrayList]@()
-$shadersA2C = [System.Collections.ArrayList]@()
+$shadersA2C = @{
+	0 = [System.Collections.ArrayList]@();
+	2 = [System.Collections.ArrayList]@();
+	4 = [System.Collections.ArrayList]@();
+	8 = [System.Collections.ArrayList]@();
+	16 = [System.Collections.ArrayList]@();
+}
 
 function Add-Shader {
 	[CmdletBinding()]
@@ -18,34 +24,38 @@ function Add-Shader {
 		[string[]]$FxcArgs
 	)
 	$hash = $HashString.Split("-") | % { [System.Convert]::ToUInt32($_, 16) }
-	$a2cstr = if ($AlphaToCoverage) { "_a2c" } else { "" }
-	$hashDecl = "0x{0:x8}, 0x{1:x8}, 0x{2:x8}, 0x{3:x8}" -f $hash[0], $hash[1], $hash[2], $hash[3]
-	$name = "shader${a2cstr}_{0:x8}_{1:x8}_{2:x8}_{3:x8}" -f $hash[0], $hash[1], $hash[2], $hash[3]
-	$decl = "{{$hashDecl}, $name, sizeof($name)}"
-	$path = "Shaders\$name.fxc"
+	$variants = if ($AlphaToCoverage) { 0, 2, 4, 8, 16 } else { 0 }
+	foreach ($variant in $variants) {
+		$a2cstr = if ($AlphaToCoverage) { "_a2c$variant" } else { "" }
+		$hashDecl = "0x{0:x8}, 0x{1:x8}, 0x{2:x8}, 0x{3:x8}" -f $hash[0], $hash[1], $hash[2], $hash[3]
+		$name = "shader${a2cstr}_{0:x8}_{1:x8}_{2:x8}_{3:x8}" -f $hash[0], $hash[1], $hash[2], $hash[3]
+		$decl = "{{$hashDecl}, $name, sizeof($name)}"
+		$path = "Shaders\$name.fxc"
+		$extra = if ($AlphaToCoverage) { "/DMSAA_SAMPLE_COUNT=$variant" } else { "" }
 
-	fxc /T ps_5_0 /Fo $path $FxcArgs $File
-	$content = [System.Io.File]::ReadAllBytes($path)
-	$array = "{"
+		fxc /T ps_5_0 /Fo $path $FxcArgs $extra $File
+		$content = [System.Io.File]::ReadAllBytes($path)
+		$array = "{"
 
-	for ($i = 0; $i -lt $content.Length; $i += 4) {
-		$num = [UInt32]$content[$i] -bor ([UInt32]$content[$i + 1] -shl 8) -bor ([UInt32]$content[$i + 2] -shl 16) -bor ([UInt32]$content[$i + 3] -shl 24)
-		if ($i % 32 -eq 0) {
-			$array += "`n  "
-		} else {
-			$array += " "
+		for ($i = 0; $i -lt $content.Length; $i += 4) {
+			$num = [UInt32]$content[$i] -bor ([UInt32]$content[$i + 1] -shl 8) -bor ([UInt32]$content[$i + 2] -shl 16) -bor ([UInt32]$content[$i + 3] -shl 24)
+			if ($i % 32 -eq 0) {
+				$array += "`n  "
+			} else {
+				$array += " "
+			}
+			$array += "0x{0:x8}," -f $num
 		}
-		$array += "0x{0:x8}," -f $num
-	}
 
-	$array += "`n};"
+		$array += "`n};"
 
-	Add-Content -NoNewline -Path Shaders.cpp "constexpr static DWORD $name[] = $array`n"
+		Add-Content -NoNewline -Path Shaders.cpp "constexpr static DWORD $name[] = $array`n"
 
-	if ($AlphaToCoverage) {
-		$shadersA2C.Add($decl)
-	} else {
-		$shaders.Add($decl)
+		if ($AlphaToCoverage) {
+			$shadersA2C[$variant].Add($decl)
+		} else {
+			$shaders.Add($decl)
+		}
 	}
 }
 
@@ -79,6 +89,10 @@ function Add-ShaderList($name, $array) {
 Add-Content -NoNewline -Path Shaders.cpp "`n`n"
 
 Add-ShaderList shaderReplacements $shaders
-Add-ShaderList shaderReplacementsAlphaToCoverage $shadersA2C
+Add-ShaderList shaderReplacementsAlphaToCoverage0 $shadersA2C[0]
+Add-ShaderList shaderReplacementsAlphaToCoverage2 $shadersA2C[2]
+Add-ShaderList shaderReplacementsAlphaToCoverage4 $shadersA2C[4]
+Add-ShaderList shaderReplacementsAlphaToCoverage8 $shadersA2C[8]
+Add-ShaderList shaderReplacementsAlphaToCoverage16 $shadersA2C[16]
 
 Add-Content -NoNewline -Path Shaders.cpp "} // namespace atfix`n"
diff --git a/impl.cpp b/impl.cpp
@@ -891,10 +891,14 @@ class DeviceWrapper final : public ID3D11Device, IDXGIDevice1 {
       bytecode = buffer.data;
       length = buffer.length;
     }
-    if (config.msaaSamples > 1 && shouldUseSampleRate(pShaderBytecode, BytecodeLength))
+    if (config.msaaSamples > 1 && shouldUseSampleRate(pShaderBytecode, BytecodeLength)) {
       converted = convertShaderToSampleRate(bytecode, length);
+    } else if (config.msaaSamples > 1 && config.sampleRateAlpha && (buffer = getAlphaToCoverageShader(pShaderBytecode, BytecodeLength)).data) {
+      bytecode = buffer.data;
+      length = buffer.length;
+    }
     HRESULT res = dev->CreatePixelShader(converted ? converted : bytecode, length, pClassLinkage, ppPixelShader);
-    if (config.msaaSamples > 1 && !converted && SUCCEEDED(res) && (buffer = getAlphaToCoverageShader(pShaderBytecode, BytecodeLength)).data) {
+    if (config.msaaSamples > 1 && !config.sampleRateAlpha && !converted && SUCCEEDED(res) && (buffer = getAlphaToCoverageShader(pShaderBytecode, BytecodeLength)).data) {
       ID3D11PixelShader* a2cShader;
       if (SUCCEEDED(dev->CreatePixelShader(buffer.data, buffer.length, pClassLinkage, &a2cShader))) {
         (*ppPixelShader)->SetPrivateDataInterface(IID_AlphaToCoverage, a2cShader);
diff --git a/impl.h b/impl.h
@@ -15,7 +15,7 @@ extern struct Config {
 	DWORD msaaSamples;
 	DWORD anisotropy;
 	bool ssaaCharacters;
-	bool ssaaTransparentObjects;
+	bool sampleRateAlpha;
 	bool ssaaAll;
 	bool allowShaderToggle;
 } config;
diff --git a/main.cpp b/main.cpp
@@ -72,8 +72,8 @@ D3D11Proc loadSystemD3D11() {
                       "NumSamples = 8\n"
                       "; Whether to do SSAA on characters (fairly cheap, improves thin lines on clothing)\n"
                       "CharacterSSAA = 1\n"
-                      "; Whether to do SSAA on transparent objects like grass and tree leaves (somewhat expensive, but prevents them from shimmering when the camera moves)\n"
-                      "ObjectSSAA = 0\n"
+                      "; Whether to use sample rate calculations for alpha of transparent objects like grass and tree leaves (somewhat cheap, reduces shimmering when the camera moves)\n"
+                      "AlphaSSAA = 1\n"
                       "; Apply SSAA to everything, because you have more GPU power than you know what to do with\n"
                       "FullSSAA = 0\n"
                       "[Other]\n"
@@ -89,26 +89,26 @@ D3D11Proc loadSystemD3D11() {
   } else {
     char NumSamples[8];
     char CharacterSSAA[8];
-    char ObjectSSAA[8];
+    char SampleRateAlpha[8];
     char FullSSAA[8];
     char AF[8];
     char UseShaderToggle[8];
     bool ok = true;
     GetPrivateProfileStringA("MSAA", "NumSamples", "8", NumSamples, sizeof(NumSamples), ".\\atfix.ini");
     GetPrivateProfileStringA("MSAA", "CharacterSSAA", "1", CharacterSSAA, sizeof(CharacterSSAA), ".\\atfix.ini");
-    GetPrivateProfileStringA("MSAA", "ObjectSSAA", "0", ObjectSSAA, sizeof(ObjectSSAA), ".\\atfix.ini");
+    GetPrivateProfileStringA("MSAA", "AlphaSSAA", "1", SampleRateAlpha, sizeof(SampleRateAlpha), ".\\atfix.ini");
     GetPrivateProfileStringA("MSAA", "FullSSAA", "0", FullSSAA, sizeof(FullSSAA), ".\\atfix.ini");
     GetPrivateProfileStringA("Other", "AF", "0", AF, sizeof(AF), ".\\atfix.ini");
     GetPrivateProfileStringA("Other", "EnhancementToggle", "0", UseShaderToggle, sizeof(UseShaderToggle), ".\\atfix.ini");
     config.msaaSamples = atoi(NumSamples);
     config.ssaaCharacters = atoi(CharacterSSAA);
-    config.ssaaTransparentObjects = atoi(ObjectSSAA);
+    config.sampleRateAlpha = atoi(SampleRateAlpha);
     config.ssaaAll = atoi(FullSSAA);
     config.anisotropy = atoi(AF);
     config.allowShaderToggle = atoi(UseShaderToggle);
     if (config.msaaSamples < 1)
       config.msaaSamples = 1;
-    log("Loaded config, ", config.msaaSamples, " samples, ", config.ssaaCharacters, " characterSSAA, ", config.ssaaTransparentObjects, " objectSSAA, ", config.ssaaAll, " fullSSAA, ", config.anisotropy, " AF, ", config.allowShaderToggle, " enhancementToggle ");
+    log("Loaded config, ", config.msaaSamples, " samples, ", config.ssaaCharacters, " characterSSAA, ", config.sampleRateAlpha, " sampleRateAlpha, ", config.ssaaAll, " fullSSAA, ", config.anisotropy, " AF, ", config.allowShaderToggle, " enhancementToggle ");
   }
 
   d3d11Proc.D3D11CreateDevice = reinterpret_cast<PFN_D3D11CreateDevice>(
