User/ivberg/fix ctf folder open upgrade sdk (#20)

Support for CTF Folder open based on new SDK NuGet Preview that supports folder open.
2021-06-28 11:27:31 -07:00 · 2021-06-28 11:27:31 -07:00 · 3083df85d0
--- a/LTTngCds/LTTngCds.csproj
+++ b/LTTngCds/LTTngCds.csproj
@ -15,7 +15,7 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LTTngCds/LTTngDataSource.cs
+++ b/LTTngCds/LTTngDataSource.cs
@ -15,6 +15,7 @@ namespace LTTngCds
        "LTTng",
        "Processes LTTng CTF data")]
    [FileDataSource("ctf", "ctf")]
+    [DirectoryDataSource("LTTng CTF Folder")]
    public class LTTngDataSource
        : CustomDataSourceBase
    {
@ -23,11 +24,14 @@ namespace LTTngCds
        /// <inheritdoc />
        public override IEnumerable<Option> CommandLineOptions => Enumerable.Empty<Option>();

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return StringComparer.OrdinalIgnoreCase.Equals(
-                ".ctf",
-                Path.GetExtension(path));
+            if (dataSource.IsDirectory())
+            {
+                return Directory.GetFiles(dataSource.Uri.LocalPath, "metadata", SearchOption.AllDirectories).Any();
+            }
+
+            return dataSource.IsFile() && StringComparer.OrdinalIgnoreCase.Equals(".ctf", Path.GetExtension(dataSource.Uri.LocalPath));
        }

        public override CustomDataSourceInfo GetAboutInfo()
@ -68,8 +72,9 @@ namespace LTTngCds

            var sourceParser = new LTTngSourceParser();

-            string sourcePath = dataSources.First().GetUri().LocalPath;
-            if (Directory.Exists(sourcePath))
+            var firstDataSource = dataSources.First();
+            string sourcePath = firstDataSource.Uri.LocalPath;
+            if (firstDataSource.IsDirectory() && Directory.Exists(sourcePath))
            {
                // handle open directory
                sourceParser.SetFolderInput(sourcePath);
--- a/LTTngDataExtUnitTest/LTTngDataExtUnitTest.csproj
+++ b/LTTngDataExtUnitTest/LTTngDataExtUnitTest.csproj
@ -8,9 +8,9 @@

  <ItemGroup>
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
-    <PackageReference Include="Microsoft.Performance.SDK.Runtime" Version="0.108.2" />
-    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
+    <PackageReference Include="Microsoft.Performance.SDK.Runtime" Version="0.109.11-preview-g61bc0d83f6" />
+    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.109.11-preview-g61bc0d83f6" />
    <PackageReference Include="Moq" Version="4.16.1" />
    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
--- a/LTTngDataExtUnitTest/LTTngUnitTest.cs
+++ b/LTTngDataExtUnitTest/LTTngUnitTest.cs
@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using LTTngCds;
 using LTTngDataExtensions.SourceDataCookers;
 using LTTngDataExtensions.DataOutputTypes;
 using LTTngDataExtensions.SourceDataCookers.Syscall;
@ -13,12 +12,12 @@ using Microsoft.Performance.SDK.Extensibility;
 using Microsoft.Performance.SDK.Processing;
 using Microsoft.Performance.Toolkit.Engine;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Moq;
 using UnitTestCommon;
 using LTTngDataExtensions.SourceDataCookers.Diagnostic_Messages;
-using Microsoft.Performance.SDK;
 using LTTngDataExtensions.SourceDataCookers.Module;
 using LTTngDataExtensions.SourceDataCookers.Disk;
+using System.IO.Compression;
+using System.Linq;

 namespace LTTngDataExtUnitTest
 {
@ -89,6 +88,29 @@ namespace LTTngDataExtUnitTest
            }
        }

+        [TestMethod]
+        public void ProcessTraceAsFolder()
+        {
+            // Input data
+            string[] lttngData = { @"..\..\..\..\TestData\LTTng\lttng-kernel-trace.ctf" };
+
+            string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+
+            ZipFile.ExtractToDirectory(lttngData[0], tempDirectory);
+
+            // Approach #1 - Engine - Doesn't test tables UI but tests processing
+            var runtime = Engine.Create();
+
+            var ds = new DirectoryDataSource(tempDirectory);
+            runtime.AddDataSource(ds);
+
+            Assert.IsTrue(ds.IsDirectory());
+            Assert.IsTrue(runtime.SourceDataCookers.Count() >= 1);
+            Assert.IsTrue(runtime.AvailableTables.Count() >= 1);
+
+            Directory.Delete(tempDirectory, true);
+        }
+
        [TestMethod]
        public void DiagnosticMessageTable()
        {
--- a/LTTngDataExtensions/LTTngDataExtensions.csproj
+++ b/LTTngDataExtensions/LTTngDataExtensions.csproj
@ -50,7 +50,7 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LTTngDriver/LTTngDriver.csproj
+++ b/LTTngDriver/LTTngDriver.csproj
@ -6,8 +6,8 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
-    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
+    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LTTngDriver/Program.cs
+++ b/LTTngDriver/Program.cs
@ -36,6 +36,9 @@ namespace LTTngDriver
                return false;
            }

+            // Debug
+            //Console.ReadLine();
+
            //
            // Create our runtime environment, enabling cookers and
            // adding inputs.
--- a/LinuxLogParsers/LinuxLogParser/LinuxLogParser.csproj
+++ b/LinuxLogParsers/LinuxLogParser/LinuxLogParser.csproj
@ -5,7 +5,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LinuxLogParsers/LinuxLogParserCore/LinuxLogParserCore.csproj
+++ b/LinuxLogParsers/LinuxLogParserCore/LinuxLogParserCore.csproj
@ -5,7 +5,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

 </Project>
--- a/LinuxLogParsers/LinuxLogParsersUnitTest/LinuxLogParsersUnitTest.csproj
+++ b/LinuxLogParsers/LinuxLogParsersUnitTest/LinuxLogParsersUnitTest.csproj
@ -8,8 +8,8 @@

  <ItemGroup>
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
-    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
+    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.109.11-preview-g61bc0d83f6" />
    <PackageReference Include="Moq" Version="4.16.1" />
    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/Cloud-Init.csproj
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/Cloud-Init.csproj
@ -9,7 +9,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/CloudInitCustomDataSource.cs
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/Cloud-init/CloudInitCustomDataSource.cs
@ -52,11 +52,11 @@ namespace CloudInitMPTAddin
            this.applicationEnvironment = applicationEnvironment;
        }

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return StringComparer.OrdinalIgnoreCase.Equals(
+            return dataSource.IsFile() && StringComparer.OrdinalIgnoreCase.Equals(
                "cloud-init.log",
-                Path.GetFileName(path));
+                Path.GetFileName(dataSource.Uri.LocalPath));
        }

        protected override ICustomDataProcessor CreateProcessorCore(
@ -64,7 +64,7 @@ namespace CloudInitMPTAddin
            IProcessorEnvironment processorEnvironment,
            ProcessorOptions options)
        {
-            string[] filePaths = dataSources.Select(x => x.GetUri().LocalPath).ToArray();
+            string[] filePaths = dataSources.Select(x => x.Uri.LocalPath).ToArray();
            var sourceParser = new CloudInitLogParser(filePaths);

            return new CloudInitCustomDataProcessor(
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/Dmesg.csproj
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/Dmesg.csproj
@ -6,7 +6,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/DmesgIsoDataSource.cs
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/DmesgIsoLog/DmesgIsoDataSource.cs
@ -21,7 +21,7 @@ namespace DmesgIsoMPTAddin

        protected override ICustomDataProcessor CreateProcessorCore(IEnumerable<IDataSource> dataSources, IProcessorEnvironment processorEnvironment, ProcessorOptions options)
        {
-            string[] filePaths = dataSources.Select(x => x.GetUri().LocalPath).ToArray();
+            string[] filePaths = dataSources.Select(x => x.Uri.LocalPath).ToArray();
            var sourceParser = new DmesgIsoLogParser(filePaths);

            return new DmesgIsoCustomDataProcessor(
@ -33,9 +33,9 @@ namespace DmesgIsoMPTAddin
                this.MetadataTables);
        }

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return path.ToLower().EndsWith("dmesg.iso.log");
+            return dataSource.IsFile() && dataSource.Uri.LocalPath.ToLower().EndsWith("dmesg.iso.log");
        }

        protected override void SetApplicationEnvironmentCore(IApplicationEnvironment applicationEnvironment)
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/WaLinuxAgent/WaLinuxAgent.csproj
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/WaLinuxAgent/WaLinuxAgent.csproj
@ -8,7 +8,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/WaLinuxAgent/WaLinuxAgentCustomDataSource.cs
+++ b/LinuxLogParsers/LinuxPlugins-MicrosoftPerformanceToolkSDK/WaLinuxAgent/WaLinuxAgentCustomDataSource.cs
@ -53,11 +53,9 @@ namespace WaLinuxAgentMPTAddin
            this.applicationEnvironment = applicationEnvironment;
        }

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return StringComparer.OrdinalIgnoreCase.Equals(
-                "waagent.log",
-                Path.GetFileName(path));
+            return dataSource.IsFile() && StringComparer.OrdinalIgnoreCase.Equals("waagent.log", Path.GetFileName(dataSource.Uri.LocalPath));
        }

        protected override ICustomDataProcessor CreateProcessorCore(
@ -65,7 +63,7 @@ namespace WaLinuxAgentMPTAddin
            IProcessorEnvironment processorEnvironment,
            ProcessorOptions options)
        {
-            string[] filePaths = dataSources.Select(x => x.GetUri().LocalPath).ToArray();
+            string[] filePaths = dataSources.Select(x => x.Uri.LocalPath).ToArray();
            var sourceParser = new WaLinuxAgentLogParser(filePaths);

            return new WaLinuxAgentCustomDataProcessor(
--- a/PerfCds/PerfCds.csproj
+++ b/PerfCds/PerfCds.csproj
@ -15,7 +15,7 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/PerfCds/PerfDataSource.cs
+++ b/PerfCds/PerfDataSource.cs
@ -15,6 +15,7 @@ namespace PerfCds
        "Perf",
        "Processes Perf CTF data")]
    [FileDataSource("ctf", "ctf")]
+    [DirectoryDataSource("Perf CTF Folder")]
    public class PerfDataSource
        : CustomDataSourceBase
    {
@ -23,11 +24,14 @@ namespace PerfCds
        /// <inheritdoc />
        public override IEnumerable<Option> CommandLineOptions => Enumerable.Empty<Option>();

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return StringComparer.OrdinalIgnoreCase.Equals(
-                ".ctf",
-                Path.GetExtension(path));
+            if (dataSource.IsDirectory())
+            {
+                return Directory.GetFiles(dataSource.Uri.LocalPath, "metadata", SearchOption.AllDirectories).Any();
+            }
+            
+            return dataSource.IsFile() && StringComparer.OrdinalIgnoreCase.Equals(".ctf", Path.GetExtension(dataSource.Uri.LocalPath));
        }

        public override CustomDataSourceInfo GetAboutInfo()
@ -68,7 +72,7 @@ namespace PerfCds

            var sourceParser = new PerfSourceParser();

-            string sourcePath = dataSources.First().GetUri().LocalPath;
+            string sourcePath = dataSources.First().Uri.LocalPath;
            if (Directory.Exists(sourcePath))
            {
                // handle open directory
--- a/PerfDataExtensions/PerfDataExtensions.csproj
+++ b/PerfDataExtensions/PerfDataExtensions.csproj
@ -48,10 +48,10 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="2.0.57" GeneratePathProperty="true">
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="2.0.66" GeneratePathProperty="true">
      <IncludeAssets>compile</IncludeAssets>
    </PackageReference>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

  <ItemGroup>
--- a/PerfDataExtensions/SourceDataCookers/PerfDataCustomDataSource.cs
+++ b/PerfDataExtensions/SourceDataCookers/PerfDataCustomDataSource.cs
@ -59,11 +59,9 @@ namespace PerfDataCustomDataSource
            this.applicationEnvironment = applicationEnvironment;
        }

-        protected override bool IsFileSupportedCore(string path)
+        protected override bool IsDataSourceSupportedCore(IDataSource dataSource)
        {
-            return StringComparer.OrdinalIgnoreCase.Equals(
-                "perf.data.txt",
-                Path.GetFileName(path));
+            return dataSource.IsFile() && StringComparer.OrdinalIgnoreCase.Equals("perf.data.txt", Path.GetFileName(dataSource.Uri.LocalPath));
        }

        protected override ICustomDataProcessor CreateProcessorCore(
@ -78,7 +76,7 @@ namespace PerfDataCustomDataSource
            //

            return new PerfDataCustomDataProcessor(
-                dataSources.Select(x => x.GetUri().LocalPath).ToArray(),
+                dataSources.Select(x => x.Uri.LocalPath).ToArray(),
                options,
                this.applicationEnvironment,
                processorEnvironment,
--- a/PerfUnitTest/PerfUnitTest.cs
+++ b/PerfUnitTest/PerfUnitTest.cs
@ -3,12 +3,14 @@

 using Microsoft.Performance.SDK;
 using Microsoft.Performance.SDK.Processing;
+using Microsoft.Performance.Toolkit.Engine;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 using PerfDataExtensions.Tables;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Threading;
 using UnitTestCommon;
@ -28,7 +30,7 @@ namespace PerfUnitTest

            var perfDataPathFullPath = perfDataPath.FullName;
            var datasource = new Mock<IDataSource>();
-            datasource.Setup(ds => ds.GetUri()).Returns(new Uri(perfDataPathFullPath));
+            datasource.Setup(ds => ds.Uri).Returns(new Uri(perfDataPathFullPath));

            // Env
            var appEnv = new Mock<IApplicationEnvironment>();
--- a/PerfUnitTest/PerfUnitTest.csproj
+++ b/PerfUnitTest/PerfUnitTest.csproj
@ -7,9 +7,10 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="2.0.57" />
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="2.0.66" />
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
+    <PackageReference Include="Microsoft.Performance.Toolkit.Engine" Version="0.109.11-preview-g61bc0d83f6" />
    <PackageReference Include="Moq" Version="4.16.1" />
    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
--- a/README.md
+++ b/README.md
@ -40,8 +40,8 @@ The tools can be run in several modes:
    - [Perf](PerfUnitTest/PerfUnitTest.cs)
    - [LinuxLogs](LinuxLogParsers/LinuxLogParsersUnitTest/LinuxLogParsersUnitTest.cs)
  - With a driver program for example dumping to screen or text format
+    - ./LTTngDriver.exe LTTngKernelTraceFolder
    - ./LTTngDriver.exe LTTng-Kernel-Trace.ctf (trace folder is zipped and renamed to .ctf)
-    - ./LTTngDriver.exe LTTngKernelTraceFolder (not currently working - blocked on [Issue #6](https://github.com/microsoft/Microsoft-Performance-Tools-Linux/issues/6))
 - (Windows Only - Install) Using the WPA GUI to load these tools as plugins
  - Download the latest Windows Performance Toolkit (WPT) that comes in the [ADK Preview](https://www.microsoft.com/en-us/software-download/windowsinsiderpreviewADK)
 	- Note: In the future, the WPA team plans to make it easier to install WPA outside of the ADK
@ -80,6 +80,7 @@ Once you gather the data, there is a tiny bit of prep needed to open them in a s

 - LTTng - If you just need to open only a LTTng trace by itself in folder format
  - WPA -> Open -> Folder (Select CTF folder)
+  - Note: Requires >= WPA ADK xxxxx - WPA 10.?.?.?
 - Unified (LTTng or other multiple different logs files together)
  - If you want to open other logs together in single timeline - Copy other Linux logs you want to open to single folder
  - Example: You want to open in the same timeline: LTTng, Perf CPU Sampling, Dmesg
--- a/UnitTestCommon/UnitTestCommon.csproj
+++ b/UnitTestCommon/UnitTestCommon.csproj
@ -5,7 +5,7 @@
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Performance.SDK" Version="0.108.2" />
+    <PackageReference Include="Microsoft.Performance.SDK" Version="0.109.11-preview-g61bc0d83f6" />
  </ItemGroup>

 </Project>