Create UDF Exctractor (#120)

* Create UDF Exctractor

It was derived IsoExtractor with some minor UDF context changes

* Update Extractor for UDF

Add UDF extractror

* Update NuGet Package

Add DiscUtils.Udf dependency

* Update MiniMagic.cs

Add enumerate to UDF

* Update MiniMagic.cs

Add enum, and signature check

* Update Tests and add Test Archives for UDF Extractor

Co-authored-by: Gabe Stocco <98900+gfs@users.noreply.github.com>

RecursiveExtractor.Tests/ExtractorTests/ExpectedNumFilesTests.cs

@@ -37,7 +37,9 @@ namespace RecursiveExtractor.Tests.ExtractorTests
                     new object[] { "TestData.vhdx",3 },
                     new object[] { "TestData.wim",3 },
                     new object[] { "EmptyFile.txt", 1 },
-                    new object[] { "TestDataArchivesNested.Zip", 54 }
+                    new object[] { "TestDataArchivesNested.Zip", 54 },
+                    new object[] { "UdfTest.iso", 3 },
+                    new object[] { "UdfTestWithMultiSystem.iso", 3 }
                 };
             }
         }
@@ -170,7 +172,11 @@ namespace RecursiveExtractor.Tests.ExtractorTests
             var extractor = new Extractor();
             var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", "TestDataArchives", fileName);
             var results = extractor.Extract(path, GetExtractorOptions()).ToList();
-            Assert.AreEqual(expectedNumFiles, results.Count());
+            foreach (var result in results)
+            {
+                Assert.AreNotEqual(FileEntryStatus.FailedArchive, result.EntryStatus);
+            }
+            Assert.AreEqual(expectedNumFiles, results.Count);
         }
         
         [TestMethod]

RecursiveExtractor.Tests/ExtractorTests/MiniMagicTests.cs

@@ -19,7 +19,7 @@ public class MiniMagicTests : BaseExtractorTestClass
     [DataRow("sysvbanner_1.0-17fakesync1_amd64.deb", ArchiveFileType.DEB)]
     [DataRow("TestData.a", ArchiveFileType.AR)]
     [DataRow("TestData.iso", ArchiveFileType.ISO_9660)]
-    //        [DataRow("TestData.vhd", ArchiveFileType.VHD)]
+    [DataRow("UdfTest.iso", ArchiveFileType.UDF)]
     [DataRow("TestData.vhdx", ArchiveFileType.VHDX)]
     [DataRow("TestData.wim", ArchiveFileType.WIM)]
     [DataRow("Empty.vmdk", ArchiveFileType.VMDK)]

RecursiveExtractor.Tests/RecursiveExtractor.Tests.csproj

@@ -267,5 +267,11 @@
     <None Update="TestData\TestData\Lorem.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="TestData\TestDataArchives\UdfTest.iso">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\TestDataArchives\UdfTestWithMultiSystem.iso">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 </Project>

RecursiveExtractor.Tests/SanitizePathTests.cs

@@ -1,18 +1,11 @@
 // Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 
+using Microsoft.CST.RecursiveExtractor;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using NLog;
-using NLog.Config;
-using NLog.Targets;
-using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Runtime.InteropServices;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 
-namespace Microsoft.CST.RecursiveExtractor.Tests
+namespace RecursiveExtractor.Tests
 {
     [TestClass]
     public class SanitizePathTests

RecursiveExtractor/Extractor.cs

@@ -51,6 +51,7 @@ namespace Microsoft.CST.RecursiveExtractor
             SetExtractor(ArchiveFileType.AR, new GnuArExtractor(this));
             SetExtractor(ArchiveFileType.GZIP, new GzipExtractor(this));
             SetExtractor(ArchiveFileType.ISO_9660, new IsoExtractor(this));
+            SetExtractor(ArchiveFileType.UDF, new UdfExtractor(this));
             SetExtractor(ArchiveFileType.RAR, new RarExtractor(this));
             SetExtractor(ArchiveFileType.RAR5, new RarExtractor(this));
             SetExtractor(ArchiveFileType.P7ZIP, new SevenZipExtractor(this));
@@ -663,4 +664,4 @@ namespace Microsoft.CST.RecursiveExtractor
             }
         }
     }
-}
+}

RecursiveExtractor/Extractors/UdfExtractor.cs

@@ -0,0 +1,157 @@
+using DiscUtils.Udf;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Microsoft.CST.RecursiveExtractor.Extractors
+{
+    /// <summary>
+    /// The UDF disc image extractor implementation.
+    /// </summary>
+    public class UdfExtractor : AsyncExtractorInterface
+    {
+        /// <summary>
+        /// The constructor takes the Extractor context for recursion.
+        /// </summary>
+        /// <param name="context">The Extractor context.</param>
+        public UdfExtractor(Extractor context)
+        {
+            Context = context;
+        }
+        private readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger();
+
+        internal Extractor Context { get; }
+
+        /// <summary>
+        ///     Extracts an UDF file
+        /// </summary>
+        ///<inheritdoc />
+        public async IAsyncEnumerable<FileEntry> ExtractAsync(FileEntry fileEntry, ExtractorOptions options, ResourceGovernor governor, bool topLevel = true)
+        {
+            DiscUtils.DiscFileInfo[]? entries = null;
+            var failed = false;
+            try
+            {
+                using var cd = new UdfReader(fileEntry.Content);
+                entries = cd.Root.GetFiles("*.*", SearchOption.AllDirectories).ToArray();
+            }
+            catch (Exception e)
+            {
+                Logger.Debug("Failed to open UDF {0}. ({1}:{2})", fileEntry.FullPath, e.GetType(), e.Message);
+                failed = true;
+            }
+            if (failed)
+            {
+                if (options.ExtractSelfOnFail)
+                {
+                    fileEntry.EntryStatus = FileEntryStatus.FailedArchive;
+                    yield return fileEntry;
+                }
+            }
+            else if (entries != null)
+            {
+                foreach (var file in entries)
+                {
+                    var fileInfo = file;
+                    governor.CheckResourceGovernor(fileInfo.Length);
+                    Stream? stream = null;
+                    try
+                    {
+                        stream = fileInfo.OpenRead();
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Debug("Failed to extract {0} from UDF {1}. ({2}:{3})", fileInfo.FullName, fileEntry.FullPath, e.GetType(), e.Message);
+                    }
+                    if (stream != null)
+                    {
+                        var name = fileInfo.FullName.Replace('/', Path.DirectorySeparatorChar);
+                        var newFileEntry = await FileEntry.FromStreamAsync(name, stream, fileEntry, fileInfo.CreationTime, fileInfo.LastWriteTime, fileInfo.LastAccessTime, memoryStreamCutoff: options.MemoryStreamCutoff).ConfigureAwait(false);
+                        if (options.Recurse || topLevel)
+                        {
+                            await foreach (var entry in Context.ExtractAsync(newFileEntry, options, governor, false))
+                            {
+                                yield return entry;
+                            }
+                        }
+                        else
+                        {
+                            yield return newFileEntry;
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        ///     Extracts an UDF file
+        /// </summary>
+        ///<inheritdoc />
+        public IEnumerable<FileEntry> Extract(FileEntry fileEntry, ExtractorOptions options, ResourceGovernor governor, bool topLevel = true)
+        {
+            DiscUtils.DiscFileInfo[]? entries = null;
+            var failed = false;
+            try
+            {
+                using var cd = new UdfReader(fileEntry.Content);
+                entries = cd.Root.GetFiles("*.*", SearchOption.AllDirectories).ToArray();
+            }
+            catch(Exception e)
+            {
+                Logger.Debug("Failed to open UDF {0}. ({1}:{2})", fileEntry.FullPath, e.GetType(), e.Message);
+                failed = true;
+            }
+            if (failed)
+            {
+                if (options.ExtractSelfOnFail)
+                {
+                    fileEntry.EntryStatus = FileEntryStatus.FailedArchive;
+                    yield return fileEntry;
+                }
+            }
+            else if (entries != null)
+            {
+                foreach (var file in entries)
+                {
+                    var fileInfo = file;
+                    governor.CheckResourceGovernor(fileInfo.Length);
+                    Stream? stream = null;
+                    try
+                    {
+                        stream = fileInfo.OpenRead();
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Debug("Failed to extract {0} from UDF {1}. ({2}:{3})", fileInfo.FullName, fileEntry.FullPath, e.GetType(), e.Message);
+                    }
+                    if (stream != null)
+                    {
+                        var name = fileInfo.FullName.Replace('/', Path.DirectorySeparatorChar);
+                        var newFileEntry = new FileEntry(name, stream, fileEntry, createTime: file.CreationTime, modifyTime: file.LastWriteTime, accessTime: file.LastAccessTime, memoryStreamCutoff: options.MemoryStreamCutoff);
+                        if (options.Recurse || topLevel)
+                        {
+                            foreach (var entry in Context.Extract(newFileEntry, options, governor, false))
+                            {
+                                yield return entry;
+                            }
+                        }
+                        else
+                        {
+                            yield return newFileEntry;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                if (options.ExtractSelfOnFail)
+                {
+                    fileEntry.EntryStatus = FileEntryStatus.FailedArchive;
+                    yield return fileEntry;
+                }
+            }
+        }
+    }
+}

RecursiveExtractor/MiniMagic.cs

@@ -61,6 +61,10 @@ namespace Microsoft.CST.RecursiveExtractor
         /// </summary>
         ISO_9660,
         /// <summary>
+        /// An UDF disc image. <see cref="Extractors.UdfExtractor"/>
+        /// </summary>
+        UDF,
+        /// <summary>
         /// A VHDX disc image. <see cref="Extractors.VhdxExtractor"/>
         /// </summary>
         VHDX,
@@ -228,6 +232,10 @@ namespace Microsoft.CST.RecursiveExtractor
                 {
                     return ArchiveFileType.ISO_9660;
                 }
+                if (buffer[0] == 'B' && buffer[1] == 'E' && buffer[2] == 'A' && buffer[3] == '0' && buffer[4] == '1')
+                {
+                    return ArchiveFileType.UDF;
+                }
             }
 
             //https://www.microsoft.com/en-us/download/details.aspx?id=23850 - 'Hard Disk Footer Format'
@@ -265,4 +273,4 @@ namespace Microsoft.CST.RecursiveExtractor
         /// <returns>The ArchiveFileType detected</returns>
         public static ArchiveFileType DetectFileType(FileEntry fileEntry) => DetectFileType(fileEntry?.Content ?? new MemoryStream());
     }
-}
+}

RecursiveExtractor/RecursiveExtractor.csproj

@@ -32,6 +32,7 @@
     <PackageReference Include="DiscUtils.HfsPlus" Version="0.16.13" />
     <PackageReference Include="DiscUtils.Iso9660" Version="0.16.13" />
     <PackageReference Include="DiscUtils.Ntfs" Version="0.16.13" />
+    <PackageReference Include="DiscUtils.Udf" Version="0.16.13" />
     <PackageReference Include="DiscUtils.Vhd" Version="0.16.13" />
     <PackageReference Include="DiscUtils.Vhdx" Version="0.16.13" />
     <PackageReference Include="DiscUtils.Vmdk" Version="0.16.13" />