Added support for brotli and gzip pre-compression

2019-02-07 15:26:11 -05:00 · 2019-02-07 15:26:11 -05:00 · 43b90bf8c0
--- a/Readme.md
+++ b/Readme.md
@ -173,6 +173,25 @@ Adding assemblies to this list will exclude them from being compiled to WebAssem
 	- `cd emscripten/1.38.13; patch -N -p1 < fix-emscripten-7399.diff`

 ## Features
+### Support for IIS / Azure Webapp GZip/Brotli pre-compression
+The IIS compression support has too many knobs for the size of generated WebAssembly files, which
+makes the serving of static files inefficient.
+
+The Bootstrapper tooling will generate two folders `_compressed_gz` and `_compressed_br` which contain compressed versions of the main files. A set IIS rewriting rules are used to redirect the queries to the requested pre-compressed files, with a preference for Brotli.
+
+When building an application, place [the following file](src/Uno.Wasm.Sample/wwwroot/web.config) in the `wwwroot` folder to automatically enable the use of pre-compressed files.
+
+The parameters for the compression are as follows:
+- `WasmShellGenerateCompressedFiles` which can be `true` of `false`. This property is ignored when building `MonoRuntimeDebuggerEnabled` is set to `true`.
+- `WasmShellCompressedExtension` is an item group which specifies which files to compress. By default `wasm`, `clr`, `js`, `css` and `html files are pre-compressed. More files can be added as follows:
+```xml
+  <ItemGroup>
+    <WasmShellCompressedExtension Include=".db"/>
+  </ItemGroup>
+```
+
+Note that the pre-compressed files are optional, and if the rewriting rules are removed or not used (because the site is served without IIS), the original files are available at their normal locations.
+
 ### Support for additional JS files
 Providing additional JS files is done through the inclusion of `EmbeddedResource`  msbuild item  files, in a project folder named `WasmScripts`.
 Files are processed as embedded resources to allow for libraries to provide javascript files.
--- a/src/Uno.Wasm.AotTests/Uno.Wasm.AotTests.csproj
+++ b/src/Uno.Wasm.AotTests/Uno.Wasm.AotTests.csproj
@ -18,7 +18,6 @@
  <ItemGroup>
 	  <None Remove="Uno.Wasm.AotTests.xml" />
 	  <None Remove="WasmScripts\image.js" />
-	  <None Remove="web.config" />
 	</ItemGroup>

  <ItemGroup>
@ -30,6 +29,12 @@
 		</EmbeddedResource>
 	  <EmbeddedResource Include="WasmScripts\image.js" />
 	</ItemGroup>
+
+	<ItemGroup>
+	  <Content Include="..\Uno.Wasm.Sample\wwwroot\web.config">
+	    <Link>wwwroot\web.config</Link>
+	  </Content>
+	</ItemGroup>
 	
 	<ItemGroup>
 		<ProjectReference Include="..\Uno.Wasm.Bootstrap.Cli\Uno.Wasm.Bootstrap.Cli.csproj">
--- a/src/Uno.Wasm.AotTests/web.config
+++ b/src/Uno.Wasm.AotTests/web.config
@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-	<system.web>
-		<customErrors mode="Off"/>
-	</system.web>
-
-	<system.webServer>
-		<urlCompression doStaticCompression="true" doDynamicCompression="true" />
-
-		<staticContent>
-			<mimeMap fileExtension=".wasm" mimeType="application/wasm" />
-			<mimeMap fileExtension=".clr" mimeType="application/octet-stream" />
-			<mimeMap fileExtension=".woff2" mimeType="application/woff2" />
-		</staticContent>
-
-		<httpCompression maxDiskSpaceUsage="500"
-        noCompressionForRange="false" 
-        doDiskSpaceLimiting="false"
-				dynamicCompressionDisableCpuUsage="100"
-				dynamicCompressionEnableCpuUsage="0"
-        staticCompressionEnableCpuUsage="0" 
-				staticCompressionDisableCpuUsage="100"
-        staticCompressionIgnoreHitFrequency="true">
-      <dynamicTypes>
-        <clear />
-        <add enabled="true" mimeType="application/wasm"/>
-        <add enabled="true" mimeType="application/octet-stream"/>
-        <add enabled="false" mimeType="*/*"/>
-      </dynamicTypes>
-      <staticTypes>
-        <clear />
-        <add enabled="true" mimeType="application/wasm"/>
-        <add enabled="true" mimeType="application/octet-stream"/>
-        <add enabled="false" mimeType="*/*"/>
-      </staticTypes>
-    </httpCompression>
-	</system.webServer>
-</configuration>
--- a/src/Uno.Wasm.Bootstrap/ShellTask.cs
+++ b/src/Uno.Wasm.Bootstrap/ShellTask.cs
@ -87,6 +87,10 @@ namespace Uno.Wasm.Bootstrap

 		public Microsoft.Build.Framework.ITaskItem[] MixedModeExcludedAssembly { get; set; }

+		public Microsoft.Build.Framework.ITaskItem[] CompressedExtensions { get; set; }
+
+		public bool GenerateCompressedFiles { get; set; }
+
 		[Microsoft.Build.Framework.Required]
 		public string RuntimeConfiguration { get; set; }

@ -122,6 +126,8 @@ namespace Uno.Wasm.Bootstrap
 				ExtractAdditionalCSS();
 				GenerateHtml();
 				GenerateConfig();
+				TryCompressDist();
+
 				return true;
 			}
 			catch (Exception ex)
@ -131,6 +137,100 @@ namespace Uno.Wasm.Bootstrap
 			}
 		}

+		private void TryCompressDist()
+		{
+			var hasCompressedExtensions = (CompressedExtensions?.Any() ?? false);
+
+			if (
+				!RuntimeDebuggerEnabled
+				&& GenerateCompressedFiles
+				&& hasCompressedExtensions
+			)
+			{
+				var compressibleExtensions = CompressedExtensions
+					.Select(e => e.ItemSpec);
+
+				Log.LogMessage(MessageImportance.Low, $"Compressing {string.Join(", ", compressibleExtensions)}");
+
+				var filesToCompress = compressibleExtensions
+					.SelectMany(e => Directory.GetFiles(_distPath, "*" + e, SearchOption.AllDirectories))
+					.Where(f => !Path.GetDirectoryName(f).Contains("_compressed_"))
+					.Distinct()
+					.ToArray();
+
+				CompressFiles(filesToCompress, "gz", GzipCompress);
+				CompressFiles(filesToCompress, "br", BrotliCompress);
+			}
+			else
+			{
+				Log.LogMessage(MessageImportance.Low,
+					$"Compression is disabled (RuntimeDebuggerEnabled:{RuntimeDebuggerEnabled}, " +
+					$"GenerateCompressedFiles:{GenerateCompressedFiles}, " +
+					$"hasCompressedExtensions:{hasCompressedExtensions})");
+			}
+		}
+
+		private void CompressFiles(string[] filesToCompress, string method, Action<string, string> compressHandler)
+		{
+			filesToCompress
+				.AsParallel()
+				.Select(fileName =>
+				{
+					var compressedPathBase = Path.Combine(_distPath, "_compressed_" + method);
+
+					var compressedFileName = fileName;
+					compressedFileName = compressedFileName.Replace(_distPath, compressedPathBase);
+
+					Directory.CreateDirectory(Path.GetDirectoryName(compressedFileName));
+
+					if (File.Exists(compressedFileName))
+					{
+						if (File.GetCreationTime(compressedFileName) < File.GetCreationTime(fileName))
+						{
+							Log.LogMessage(MessageImportance.High, $"Deleting {compressedFileName} as the source has changed");
+							File.Delete(compressedFileName);
+						}
+					}
+
+					Log.LogMessage(MessageImportance.Low, $"Compressing {fileName}->{compressedFileName}");
+
+					compressHandler(fileName, compressedFileName);
+
+					return true;
+				})
+				.ToArray();
+		}
+
+		private void GzipCompress(string source, string destination)
+		{
+			using (var inStream = File.Open(source, FileMode.Open, FileAccess.Read, FileShare.Read))
+			using (var compressedFileStream = File.Create(destination))
+			using (var compressionStream = new GZipStream(compressedFileStream, CompressionLevel.Optimal))
+			{
+				inStream.CopyTo(compressionStream);
+			}
+		}
+
+		private void BrotliCompress(string source, string destination)
+		{
+			using (var input = File.Open(source, FileMode.Open, FileAccess.Read, FileShare.Read))
+			using (var output = File.Create(destination))
+			using (var bs = new BrotliSharpLib.BrotliStream(output, CompressionMode.Compress))
+			{
+				// By default, BrotliSharpLib uses a quality value of 1 and window size of 22 if the methods are not called.
+				bs.SetQuality(11);
+				/** bs.SetWindow(windowSize); **/
+				/** bs.SetCustomDictionary(customDict); **/
+				input.CopyTo(bs);
+
+				/* IMPORTANT: Only use the destination stream after closing/disposing the BrotliStream
+				   as the BrotliStream must be closed in order to signal that no more blocks are being written
+				   for the final block to be flushed out 
+				*/
+				bs.Dispose();
+			}
+		}
+
 		private int RunProcess(string executable, string parameters, string workingDirectory = null)
 		{
 			var p = new Process
--- a/src/Uno.Wasm.Bootstrap/Uno.Wasm.Bootstrap.csproj
+++ b/src/Uno.Wasm.Bootstrap/Uno.Wasm.Bootstrap.csproj
@ -21,6 +21,8 @@
 	</PropertyGroup>
 	
 	<ItemGroup>
+		<PackageReference Include="BrotliSharpLib" Version="0.3.3" IncludeAssets="all" PrivateAssets="All" />
+		<PackageReference Include="system.runtime.compilerservices.unsafe" Version="4.5.2" IncludeAssets="all" PrivateAssets="All" />
 		<PackageReference Include="Microsoft.Build" Version="15.3.409" PrivateAssets="All" />
 		<PackageReference Include="Microsoft.Build.Framework" Version="15.3.409" PrivateAssets="All" />
 		<PackageReference Include="Microsoft.Build.Utilities.Core" Version="15.3.409" PrivateAssets="All" />
@ -54,6 +56,16 @@
 			<Pack>true</Pack>
 			<PackagePath>tools/templates</PackagePath>
 		</Content>
+		<Content Include="$(NuGetPackageRoot)/BrotliSharpLib/0.3.3/lib/netstandard2.0/*.dll">
+			<Pack>true</Pack>
+			<PackagePath>tools</PackagePath>
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</Content>
+		<Content Include="$(NuGetPackageRoot)/system.runtime.compilerservices.unsafe/4.5.2/lib/netstandard2.0/*.dll">
+			<Pack>true</Pack>
+			<PackagePath>tools</PackagePath>
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</Content>
 		<Content Include="$(NuGetPackageRoot)/mono.cecil/0.10.1/lib/netstandard1.3/*.dll">
 			<Pack>true</Pack>
 			<PackagePath>tools</PackagePath>
--- a/src/Uno.Wasm.Bootstrap/build/Uno.Wasm.Bootstrap.targets
+++ b/src/Uno.Wasm.Bootstrap/build/Uno.Wasm.Bootstrap.targets
@ -12,6 +12,7 @@
    <MonoWasmRuntimeConfiguration Condition="'$(WasmRuntimeConfiguration)'==''">release</MonoWasmRuntimeConfiguration>
    <MonoRuntimeDebuggerEnabled Condition="'$(MonoRuntimeDebuggerEnabled)'==''">false</MonoRuntimeDebuggerEnabled>
    <WasmShellILLinkerEnabled Condition="'$(WasmShellILLinkerEnabled)'==''">true</WasmShellILLinkerEnabled>
+    <WasmShellGenerateCompressedFiles Condition="'$(WasmShellGenerateCompressedFiles)'==''">true</WasmShellGenerateCompressedFiles>

    <!--
    Set this variable to use custom debugger binaries instead of the files found 
@ -57,6 +58,15 @@
    <RunArguments>$(_unoBinArgs) $(_unoRunArgs) --pathbase &quot;$(MSBuildProjectDirectory)\bin\$(Configuration)\$(TargetFramework)\dist&quot; --configuration $(Configuration) --targetframework $(TargetFramework) $(AdditionalRunArguments)</RunArguments>
  </PropertyGroup>

+  <ItemGroup>
+    <!-- Default compressed extensions when WasmShellGenerateCompressedFiles is enabled -->
+    <WasmShellCompressedExtension Include=".wasm"/>
+    <WasmShellCompressedExtension Include=".clr"/>
+    <WasmShellCompressedExtension Include=".js"/>
+    <WasmShellCompressedExtension Include=".css"/>
+    <WasmShellCompressedExtension Include=".html"/>
+  </ItemGroup>
+
  <UsingTask AssemblyFile="$(WasmShellTasksPath)/Uno.Wasm.Bootstrap.v0.dll" TaskName="Uno.Wasm.Bootstrap.ShellTask_v0" />
  <UsingTask AssemblyFile="$(WasmShellTasksPath)/Uno.Wasm.Bootstrap.v0.dll" TaskName="Uno.Wasm.Bootstrap.UnoInstallSDKTask_v0" />

@ -123,6 +133,8 @@
        MixedModeExcludedAssembly="@(MonoRuntimeMixedModeExcludedAssembly)"
        MonoILLinker="$(WasmShellILLinkerEnabled)"
        MonoTempFolder="$(WasmShellMonoTempFolder)"
+        GenerateCompressedFiles="$(WasmShellGenerateCompressedFiles)"
+        CompressedExtensions="@(WasmShellCompressedExtension)"
        Assets="@(Content)"
        ReferencePath="@(_AssembliesForReferenceCopyLocalPaths)" />
  </Target>
--- a/src/Uno.Wasm.Sample/Uno.Wasm.Sample.csproj
+++ b/src/Uno.Wasm.Sample/Uno.Wasm.Sample.csproj
@ -6,13 +6,9 @@
    <IsPackable>false</IsPackable>
    <StartupObject>Uno.Wasm.Sample.Program</StartupObject>
    <WasmPWAManifestFile>manifest.json</WasmPWAManifestFile>
-		<MonoRuntimeDebuggerEnabled Condition="'$(Configuration)'=='Debug'">true</MonoRuntimeDebuggerEnabled>
+		<!--<MonoRuntimeDebuggerEnabled Condition="'$(Configuration)'=='Debug'">true</MonoRuntimeDebuggerEnabled>-->
 	</PropertyGroup>

-  <ItemGroup>
-    <Folder Include="wwwroot\" />
-  </ItemGroup>
-
  <ItemGroup>
    <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
    <PackageReference Include="System.Collections.Immutable" Version="1.4.0" />
--- a/src/Uno.Wasm.Sample/wwwroot/web.config
+++ b/src/Uno.Wasm.Sample/wwwroot/web.config
@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <system.web>
+    <customErrors mode="Off"/>
+  </system.web>
+
+  <system.webServer>
+
+    <!-- Disable compression as we're doing it throuh pre-compressed files -->
+    <urlCompression doStaticCompression="false" doDynamicCompression="false" dynamicCompressionBeforeCache="false" />
+
+    <staticContent>
+      <remove fileExtension=".dll" />
+      <remove fileExtension=".wasm" />
+      <remove fileExtension=".woff" />
+      <remove fileExtension=".woff2" />
+      <mimeMap fileExtension=".wasm" mimeType="application/wasm" />
+      <mimeMap fileExtension=".clr" mimeType="application/octet-stream" />
+      <mimeMap fileExtension=".pdb" mimeType="application/octet-stream" />
+      <mimeMap fileExtension=".woff" mimeType="application/font-woff" />
+      <mimeMap fileExtension=".woff2" mimeType="application/font-woff" />
+    </staticContent>
+
+    <rewrite>
+      <rules>
+        <rule name="Lookup for pre-compressed brotli file" stopProcessing="true">
+          <match url="(.*)$"/>
+          <conditions>
+            <!-- Match brotli requests -->
+            <add input="{HTTP_ACCEPT_ENCODING}" pattern="br" />
+            
+            <!-- Match all but pre-compressed files -->
+            <add input="{REQUEST_URI}" pattern="^(?!/_compressed_br/)(.*)$" />
+
+            <!-- Check if the pre-compressed file exists on the disk -->
+            <add input="{DOCUMENT_ROOT}/_compressed_br/{C:0}" matchType="IsFile" negate="false" />
+          </conditions>
+          <action type="Rewrite" url="/_compressed_br{C:0}" />
+        </rule>
+
+        <rule name="Lookup for pre-compressed gzip file" stopProcessing="true">
+          <match url="(.*)$"/>
+          <conditions>
+            <!-- Match gzip requests -->
+            <add input="{HTTP_ACCEPT_ENCODING}" pattern="gzip" />
+            
+            <!-- Match all but pre-compressed files -->
+            <add input="{REQUEST_URI}" pattern="^(?!/_compressed_gz/)(.*)$" />
+            
+            <!-- Check if the pre-compressed file exists on the disk -->
+            <add input="{DOCUMENT_ROOT}/_compressed_gz/{C:0}" matchType="IsFile" negate="false" />
+          </conditions>
+          <action type="Rewrite" url="/_compressed_gz{C:0}" />
+        </rule>
+      </rules>
+
+      <outboundRules>
+        <rule name="Adjust content encoding for gzip pre-compressed files" enabled="true" stopProcessing="true">
+          <match serverVariable="RESPONSE_CONTENT_ENCODING" pattern="" />
+          <conditions>
+            <add input="{REQUEST_URI}" pattern="/_compressed_gz/.*$" />
+          </conditions>
+          <action type="Rewrite" value="gzip"/>
+        </rule>
+        <rule name="Adjust content encoding for brotli pre-compressed files" enabled="true" stopProcessing="true">
+          <match serverVariable="RESPONSE_CONTENT_ENCODING" pattern="" />
+          <conditions>
+            <add input="{REQUEST_URI}" pattern="/_compressed_br/.*$" />
+          </conditions>
+          <action type="Rewrite" value="br"/>
+        </rule>
+      </outboundRules>
+    </rewrite>
+  </system.webServer>
+</configuration>
--- a/src/nuget.config
+++ b/src/nuget.config
@ -0,0 +1,5 @@
+<configuration>
+  <packageSources>
+    <add key="dotnet.myget.org dotnet-corefxlab" value="https://dotnet.myget.org/F/dotnet-corefxlab/" />
+  </packageSources>
+</configuration>