Add Recursive Extractor Cli (#22)
* Add a RE Cli * Update .gitignore * Update FileEntry.cs * Add Test * Update RecursiveExtractor.Cli.csproj * Build fix * Update readme * Adds VS debugging launch setting * Fix Cli Tests * Fix ArFile sometimes leaving spare '/' on the end of filenames * Update README.md * Add a RE Cli * Update .gitignore * Update FileEntry.cs * Add Test * Update RecursiveExtractor.Cli.csproj * Build fix * Update readme * Fix Cli Tests * Fix ArFile sometimes leaving spare '/' on the end of filenames * Update README.md * Add Encrypted Archive support to Cli * Fix logging. Fix encrypted cli tests. * Fix Null reference issue * Delete test unzipped files before checking assert Previously a failed test would leave files behind (still in the temp directory). * Fix ZipSlip * ZipSlip * Update FileEntry.cs * Add ZipSlip tests * Update README.md * Update README.md * Update README.md * Add Packing CLI * Update core-pipeline.yml for Azure Pipelines * Fix Dotnet Tool Packaging * Don't attempt to run extractors that don't work on wasm * Split RAR and RAR5 archive types. * Update FileEntry.cs * Update MiniMagic.cs * Improve zipslip handling * Don't need to check for zipslip here. It uses the fileentry constructor which checks. * Print original path in FileEntry when hitting error - not reparsed ZipSlip safe path. * Fix RAR5 Minimagic test * Fix path formatting case in sevenzipextractor * Fix path formatting issue in DiscCommon * Add Allow and Deny filters to Cli These are passed as a list of regexes to either require that all files match or disallow files from matching. * Update readme with filter example Co-authored-by: David Alcantar <daalcant@microsoft.com>
This commit is contained in:
Родитель
412956c8e1
Коммит
d363524ff3
|
@ -1,7 +1,9 @@
|
|||
RecursiveExtractor/bin
|
||||
RecursiveExtractor/obj
|
||||
RecursiveExtractor.Tests/bin
|
||||
RecursiveExtractor.Tests/obj
|
||||
RecursiveExtractor.Blazor/bin
|
||||
RecursiveExtractor.Blazor/obj
|
||||
RecursiveExtractor.Cli/bin
|
||||
RecursiveExtractor.Cli/obj
|
||||
RecursiveExtractor.Tests/bin
|
||||
RecursiveExtractor.Tests/obj
|
||||
.vs
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
// Use IntelliSense to find out which attributes exist for C# debugging
|
||||
// Use hover for the description of the existing attributes
|
||||
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": ".NET Core Launch (console)",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "build",
|
||||
// If you have changed target frameworks, make sure to update the program path.
|
||||
"program": "${workspaceFolder}/RecursiveExtractor.Cli/bin/Debug/netcoreapp3.1/RecursiveExtractor.Cli.dll",
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}/RecursiveExtractor.Cli",
|
||||
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
|
||||
"console": "internalConsole",
|
||||
"stopAtEntry": false
|
||||
},
|
||||
{
|
||||
"name": ".NET Core Attach",
|
||||
"type": "coreclr",
|
||||
"request": "attach",
|
||||
"processId": "${command:pickProcess}"
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"dotnet-test-explorer.testProjectPath": "**/*Tests.csproj"
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "build",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"build",
|
||||
"--framework",
|
||||
"netcoreapp3.1",
|
||||
"${workspaceFolder}/RecursiveExtractor.Cli/RecursiveExtractor.Cli.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
},
|
||||
{
|
||||
"label": "publish",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"publish",
|
||||
"${workspaceFolder}/RecursiveExtractor.Cli/RecursiveExtractor.Cli.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
},
|
||||
{
|
||||
"label": "watch",
|
||||
"command": "dotnet",
|
||||
"type": "process",
|
||||
"args": [
|
||||
"watch",
|
||||
"run",
|
||||
"${workspaceFolder}/RecursiveExtractor.Cli/RecursiveExtractor.Cli.csproj",
|
||||
"/property:GenerateFullPaths=true",
|
||||
"/consoleloggerparameters:NoSummary"
|
||||
],
|
||||
"problemMatcher": "$msCompile"
|
||||
}
|
||||
]
|
||||
}
|
|
@ -10,6 +10,7 @@ trigger:
|
|||
paths:
|
||||
include:
|
||||
- RecursiveExtractor
|
||||
- RecursiveExtractor.Cli
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
|
@ -19,6 +20,7 @@ pr:
|
|||
- Pipelines
|
||||
- RecursiveExtractor
|
||||
- RecursiveExtractor.Blazor
|
||||
- RecursiveExtractor.Cli
|
||||
- RecursiveExtractor.Tests
|
||||
- RecursiveExtractor.sln
|
||||
|
||||
|
@ -42,9 +44,17 @@ stages:
|
|||
jobs:
|
||||
- template: templates/nuget-build-job.yml
|
||||
parameters:
|
||||
jobName: 'pack_lib'
|
||||
dotnetVersion: '3.1.x'
|
||||
projectPath: 'RecursiveExtractor/RecursiveExtractor.csproj'
|
||||
projectName: 'RecursiveExtractor'
|
||||
- template: templates/nuget-build-job-framework.yml
|
||||
parameters:
|
||||
jobName: 'pack_cli'
|
||||
dotnetVersion: '3.1.x'
|
||||
publishFramework: 'netcoreapp3.1'
|
||||
projectPath: 'RecursiveExtractor.Cli/RecursiveExtractor.Cli.csproj'
|
||||
projectName: 'RecursiveExtractor_CLI'
|
||||
|
||||
- stage: Release
|
||||
dependsOn:
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
parameters:
|
||||
# Job Name
|
||||
- name: jobName
|
||||
type: string
|
||||
default: 'nuget_pack'
|
||||
# Optional Pre-Build Steps
|
||||
- name: 'preBuild'
|
||||
type: stepList
|
||||
default: []
|
||||
# Version of Dotnet SDK to use
|
||||
- name: dotnetVersion
|
||||
type: string
|
||||
default: '3.1.x'
|
||||
# Framework version to Publish
|
||||
- name: publishFramework
|
||||
type: string
|
||||
default: 'netcoreapp3.1'
|
||||
# Version of NuGet Tool to use
|
||||
- name: nugetVersion
|
||||
type: string
|
||||
default: '5.x'
|
||||
# Path to .csproj or .sln
|
||||
- name: projectPath
|
||||
type: string
|
||||
default: ''
|
||||
# Build Configuration
|
||||
- name: buildConfiguration
|
||||
type: string
|
||||
default: 'Release'
|
||||
# Project Name
|
||||
- name: projectName
|
||||
type: string
|
||||
default: ''
|
||||
# Pipeline Artifact Name
|
||||
- name: artifactName
|
||||
type: string
|
||||
default: 'Unsigned_Binaries'
|
||||
|
||||
jobs:
|
||||
- job: ${{ parameters.jobName }}
|
||||
displayName: NuGet Package
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
steps:
|
||||
- task: UseDotNet@2
|
||||
displayName: Install Dotnet SDK
|
||||
inputs:
|
||||
packageType: 'sdk'
|
||||
version: ${{ parameters.dotnetVersion }}
|
||||
- task: NuGetToolInstaller@1
|
||||
displayName: Install Nuget Tool
|
||||
inputs:
|
||||
versionSpec: ${{ parameters.nugetVersion }}
|
||||
- ${{ parameters.preBuild }}
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: Dotnet Restore
|
||||
inputs:
|
||||
command: 'restore'
|
||||
projects: ${{ parameters.projectPath }}
|
||||
verbosityRestore: 'Normal'
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: Pack Nupkg
|
||||
inputs:
|
||||
command: 'custom'
|
||||
custom: 'pack'
|
||||
arguments: '${{ parameters.projectPath }} -c ${{ parameters.buildConfiguration }} -p:TargetFrameworks=${{ parameters.publishFramework }} -o Packages'
|
||||
- task: AntiMalware@3
|
||||
displayName: Anti-Malware Scan
|
||||
inputs:
|
||||
InputType: 'Basic'
|
||||
ScanType: 'CustomScan'
|
||||
FileDirPath: 'Packages'
|
||||
EnableServices: true
|
||||
SupportLogOnError: true
|
||||
TreatSignatureUpdateFailureAs: 'Warning'
|
||||
SignatureFreshness: 'UpToDate'
|
||||
TreatStaleSignatureAs: 'Warning'
|
||||
- task: ArchiveFiles@2
|
||||
displayName: Archive Packages
|
||||
inputs:
|
||||
rootFolderOrFile: 'Packages'
|
||||
includeRootFolder: false
|
||||
archiveType: 'zip'
|
||||
archiveFile: 'Archives\${{ parameters.projectName }}_NuGet.zip'
|
||||
replaceExistingArchive: true
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: Pipeline Publish Archive
|
||||
inputs:
|
||||
PathtoPublish: 'Archives'
|
||||
ArtifactName: '${{ parameters.artifactName }}'
|
||||
publishLocation: 'Container'
|
95
README.md
95
README.md
|
@ -17,25 +17,65 @@ You can try out Recursive Extractor [in your browser](https://microsoft.github.i
|
|||
| vhdx | vmdk | wim* |
|
||||
| xzip | zip | |
|
||||
|
||||
* Windows only
|
||||
# Usage
|
||||
This example will print out the paths of all the files in the archive.
|
||||
# Variants
|
||||
|
||||
## Browser
|
||||
You can try out Recursive Extractor [in your browser](https://microsoft.github.io/RecursiveExtractor/) as a Web Assembly app.
|
||||
|
||||
## Cli
|
||||
First ensure you have the latest [.NET SDK](https://dotnet.microsoft.com/download).
|
||||
|
||||
Then run `dotnet tool install -g Microsoft.CST.RecursiveExtractor.Cli`
|
||||
|
||||
Then you can run: `RecursiveExtractor --input archive.ext --output outputDirectory`
|
||||
<details>
|
||||
<summary>Detailed Usage</summary>
|
||||
<br/>
|
||||
<ul>
|
||||
<li><i>input</i>: The path to the Archive to extract.</li>
|
||||
<li><i>output</i>: The path a directory to extract into.</li>
|
||||
<li><i>passwords</i>: A comma separated list of passwords to use for archives.</li>
|
||||
<li><i>allow-filters</i>: A comma separated list of regexes to require each extracted file match.</li>
|
||||
<li><i>deny-filters</i>: A comma separated list of regexes to require each extracted file not match.</li>
|
||||
</ul>
|
||||
|
||||
For example, to extract only ".cs" files:
|
||||
```
|
||||
RecursiveExtractor --input archive.ext --output outputDirectory --allow-filters .cs$
|
||||
```
|
||||
|
||||
Run "RecursiveExtractor --help" for more details.
|
||||
</details>
|
||||
|
||||
## Library
|
||||
Recursive Extractor is available on NuGet as [Microsoft.CST.RecursiveExtractor](https://www.nuget.org/packages/Microsoft.CST.RecursiveExtractor/).
|
||||
|
||||
This code adapted from the Cli extracts the contents of given archive located at `options.Input`
|
||||
to a directory located at `options.Output` and prints the relative path of each file inside the archive.
|
||||
|
||||
```csharp
|
||||
var path = "/Path/To/Your/Archive"
|
||||
var extractor = new Extractor();
|
||||
try {
|
||||
IEnumerable<FileEntry> results = extractor.ExtractFile(path);
|
||||
foreach(var found in results)
|
||||
{
|
||||
Console.WriteLine(found.FullPath);
|
||||
}
|
||||
}
|
||||
catch(OverflowException)
|
||||
var extractorOptions = new ExtractorOptions()
|
||||
{
|
||||
// This means Recursive Extractor has detected a Quine or Zip Bomb
|
||||
ExtractSelfOnFail = true,
|
||||
Parallel = true,
|
||||
};
|
||||
foreach(var result in extractor.ExtractFile(options.Input, extractorOptions))
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(options.Output,
|
||||
Path.GetDirectoryName(result.FullPath)?
|
||||
.TrimStart(Path.DirectorySeparatorChar) ?? string.Empty));
|
||||
|
||||
using var fs = new FileStream(Path.Combine(options.Output,result.FullPath), FileMode.Create);
|
||||
result.Content.CopyTo(fs);
|
||||
Console.WriteLine("Extracted {0}.", result.FullPath);
|
||||
}
|
||||
```
|
||||
If you'd prefer async
|
||||
<details>
|
||||
<summary>Async Usage</summary>
|
||||
<br/>
|
||||
This example prints out all the file names found from the archive located at the path.
|
||||
|
||||
```csharp
|
||||
var path = "/Path/To/Your/Archive"
|
||||
var extractor = new Extractor();
|
||||
|
@ -51,9 +91,11 @@ catch(OverflowException)
|
|||
// This means Recursive Extractor has detected a Quine or Zip Bomb
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
## FileEntry
|
||||
|
||||
<details>
|
||||
<summary>The FileEntry Object</summary>
|
||||
<br/>
|
||||
The Extractor returns `FileEntry` objects. These objects contain a `Content` Stream of the file contents.
|
||||
|
||||
```csharp
|
||||
|
@ -63,9 +105,13 @@ public string Name { get; }
|
|||
public FileEntry? Parent { get; }
|
||||
public string? ParentPath { get; }
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Extracting Encrypted Archives</summary>
|
||||
<br/>
|
||||
The Extractor returns `FileEntry` objects. These objects contain a `Content` Stream of the file contents.You can provide passwords to use to decrypt archives, paired with a Regex that will operate against the Name of the Archive.
|
||||
|
||||
## Extracting Encrypted Archives
|
||||
You can provide passwords to use to decrypt archives, paired with a Regex that will operate against the Name of the Archive.
|
||||
```csharp
|
||||
var path = "/Path/To/Your/Archive"
|
||||
var extractor = new Extractor();
|
||||
|
@ -90,10 +136,11 @@ catch(OverflowException)
|
|||
// This means Recursive Extractor has detected a Quine or Zip Bomb
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
## Exceptions
|
||||
|
||||
`ExtractFile` will throw an overflow exception when a quine or zip bomb is detected.
|
||||
RecursiveExtractor protects against [ZipSlip](https://snyk.io/research/zip-slip-vulnerability), [Quines, and Zip Bombs](https://en.wikipedia.org/wiki/Zip_bomb).
|
||||
Calls to Extract will throw an `OverflowException` when a Quine or Zip bomb is detected.
|
||||
|
||||
Otherwise, invalid files found while crawling will emit a logger message and be skipped. RecursiveExtractor uses NLog for logging.
|
||||
|
||||
|
@ -101,6 +148,14 @@ Otherwise, invalid files found while crawling will emit a logger message and be
|
|||
|
||||
If you have any issues or feature requests please open a new [Issue](https://github.com/microsoft/RecursiveExtractor/issues/new)
|
||||
|
||||
# Dependencies
|
||||
|
||||
Recursive Extractor uses a number of libraries to parse archives.
|
||||
|
||||
* [SharpZipLib](https://github.com/icsharpcode/SharpZipLib)
|
||||
* [SharpCompress](https://github.com/adamhathcock/sharpcompress)
|
||||
* [DiscUtils](https://github.com/discutils/discutils).
|
||||
|
||||
# Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
<h2>Scan</h2>
|
||||
<p>Choose the archive that you wish to extract. Due to platform restrictions on our dependencies a limited set of formats is available in this GUI.</p>
|
||||
<h2>GUI Supported Formats</h2>
|
||||
<p>Zip, Iso, Tar, Bz, Xz, Vhdx, Wim, Ar, Deb</p>
|
||||
<p>Zip, Iso, Tar, Bz, Xz, Vhdx, Ar, Deb</p>
|
||||
<h2>Additional formats supported in library</h2>
|
||||
<p>Rar, 7z, Gz, Vhd</p>
|
||||
<p>Rar, 7z, Gz, Vhd, Wim</p>
|
||||
<DragnDropInputCommon />
|
|
@ -119,6 +119,14 @@
|
|||
Output = string.Empty;
|
||||
this.StateHasChanged();
|
||||
var extractor = new Extractor();
|
||||
|
||||
// These formats aren't supported in WASM
|
||||
extractor.Unset(ArchiveFileType.P7ZIP);
|
||||
extractor.Unset(ArchiveFileType.RAR);
|
||||
extractor.Unset(ArchiveFileType.GZIP);
|
||||
extractor.Unset(ArchiveFileType.VHD);
|
||||
extractor.Unset(ArchiveFileType.WIM);
|
||||
|
||||
var options = new ExtractorOptions() { EnableTiming = false, ExtractSelfOnFail = false };
|
||||
foreach (var pair in IpFileList)
|
||||
{
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
using CommandLine;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Microsoft.CST.RecursiveExtractor.Cli
|
||||
{
|
||||
[Verb("extract", true, HelpText = "Add file contents to the index.")]
|
||||
public class ExtractCommandOptions
|
||||
{
|
||||
[Option('i',"input",HelpText = "The name of the Archive to extract.", Required = true)]
|
||||
public string Input { get; set; } = string.Empty;
|
||||
|
||||
[Option('o', "output", HelpText = "The directory to extract to.", Default = ".")]
|
||||
public string Output { get; set; } = string.Empty;
|
||||
|
||||
[Option('p', "passwords", Required = false, HelpText = "Comma separated list of passwords to use.", Separator = ',')]
|
||||
public IEnumerable<string>? Passwords { get; set; }
|
||||
|
||||
[Option('A', "allow-filters", Required = false, HelpText = "Comma separated list of regexes. When set, files are only written to disc if they match one of these filters.", Separator = ',')]
|
||||
public IEnumerable<string>? AllowFilters { get; set; }
|
||||
|
||||
[Option('D', "deny-filters", Required = false, HelpText = "Comma separated list of regexes. When set, files are not written to disc if they match one of these filters.", Separator = ',')]
|
||||
public IEnumerable<string>? DenyFilters { get; set; }
|
||||
|
||||
[Option(HelpText = "Set logging to Verbose.")]
|
||||
public bool Verbose { get; set; }
|
||||
|
||||
[Option(HelpText = "Set logging to Debug.")]
|
||||
public bool Debug { get; set; }
|
||||
|
||||
[Option(HelpText = "Print names of all extracted files.")]
|
||||
public bool PrintNames { get; set; }
|
||||
}
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"profiles": {
|
||||
"RecursiveExtractor.Cli": {
|
||||
"commandName": "Project",
|
||||
"commandLineArgs": "--input D:\\GitHub\\RecursiveExtractor\\RecursiveExtractor.Tests\\TestData\\NestedEncrypted.7z --output D:\\RecurseOutput --verbose --passwords TheMagicWordIsPotato,TheMagicWordIsTomato,TheMagicWordIsCelery,TheMagicWordIsLettuce"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>netcoreapp3.1;net48</TargetFrameworks>
|
||||
<LangVersion>8.0</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<RootNamespace>Microsoft.CST.RecursiveExtractor.Cli</RootNamespace>
|
||||
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
|
||||
<Product>Recursive Extractor</Product>
|
||||
<Company>Microsoft</Company>
|
||||
<Authors>Microsoft</Authors>
|
||||
<Description>RecursiveExtractor is able to process the following formats: ar, bzip2, deb, gzip, iso, tar, vhd, vhdx, vmdk, wim, xzip, and zip. RecursiveExtractor automatically detects the archive type and fails gracefully when attempting to process malformed content.</Description>
|
||||
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
|
||||
<PackAsTool>true</PackAsTool>
|
||||
<PackageId>Microsoft.CST.RecursiveExtractor.CLI</PackageId>
|
||||
<PackageVersion>0.0.0</PackageVersion>
|
||||
<PackageProjectUrl>https://github.com/microsoft/RecursiveExtractor</PackageProjectUrl>
|
||||
<PackageTags>unzip extract extractor</PackageTags>
|
||||
<ToolCommandName>RecursiveExtractor</ToolCommandName>
|
||||
<PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
|
||||
<PackageIcon>icon-128.png</PackageIcon>
|
||||
<IncludeSymbols>true</IncludeSymbols>
|
||||
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\RecursiveExtractor\RecursiveExtractor.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CommandLineParser" Version="2.8.0" />
|
||||
<PackageReference Include="NLog.Extensions.Logging" Version="1.6.4" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\LICENSE.txt" Pack="true" PackagePath="" />
|
||||
<None Include="..\icon-128.png" Pack="true" PackagePath="" />
|
||||
</ItemGroup>
|
||||
</Project>
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup Condition=" '$(RunConfiguration)' == 'Default' ">
|
||||
<StartAction>Project</StartAction>
|
||||
<StartArguments>--input /Users/gabe/Documents/GitHub/RecursiveExtractor/RecursiveExtractor.Tests/TestData/Nested.zip --output /Users/gabe/Documents/GitHub/RecursiveExtractor/NestedExtracted --verbose</StartArguments>
|
||||
<ExternalConsole>true</ExternalConsole>
|
||||
</PropertyGroup>
|
||||
</Project>
|
|
@ -0,0 +1,104 @@
|
|||
using CommandLine;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using NLog;
|
||||
using NLog.Config;
|
||||
using NLog.Targets;
|
||||
namespace Microsoft.CST.RecursiveExtractor.Cli
|
||||
{
|
||||
public class RecursiveExtractorClient
|
||||
{
|
||||
public static int Main(string[] args)
|
||||
{
|
||||
return CommandLine.Parser.Default.ParseArguments<ExtractCommandOptions>(args)
|
||||
.MapResult(
|
||||
(ExtractCommandOptions opts) => ExtractCommand(opts),
|
||||
errs => 1);
|
||||
}
|
||||
|
||||
public static int ExtractCommand(ExtractCommandOptions options)
|
||||
{
|
||||
var config = new LoggingConfiguration();
|
||||
var consoleTarget = new ConsoleTarget
|
||||
{
|
||||
Name = "console",
|
||||
Layout = "${longdate}|${level:uppercase=true}|${logger}|${message}",
|
||||
};
|
||||
if (options.Verbose) {
|
||||
config.AddRule(LogLevel.Trace, LogLevel.Fatal, consoleTarget, "*");
|
||||
}
|
||||
else if (options.Debug)
|
||||
{
|
||||
config.AddRule(LogLevel.Debug, LogLevel.Fatal, consoleTarget, "*");
|
||||
}
|
||||
else
|
||||
{
|
||||
config.AddRule(LogLevel.Info, LogLevel.Fatal, consoleTarget, "*");
|
||||
}
|
||||
|
||||
LogManager.Configuration = config;
|
||||
|
||||
var extractor = new Extractor();
|
||||
var extractorOptions = new ExtractorOptions()
|
||||
{
|
||||
ExtractSelfOnFail = true,
|
||||
Parallel = true,
|
||||
};
|
||||
if (options.Passwords?.Any() ?? false)
|
||||
{
|
||||
extractorOptions.Passwords = new Dictionary<Regex, List<string>>()
|
||||
{
|
||||
{
|
||||
new Regex(".*",RegexOptions.Compiled),
|
||||
options.Passwords.ToList()
|
||||
}
|
||||
};
|
||||
}
|
||||
var allowRegexes = options.AllowFilters?.Select(x => new Regex(x)) ?? Array.Empty<Regex>();
|
||||
var denyRegexes = options.DenyFilters?.Select(x => new Regex(x)) ?? Array.Empty<Regex>();
|
||||
foreach (var result in extractor.ExtractFile(options.Input, extractorOptions))
|
||||
{
|
||||
var skip = false;
|
||||
foreach(var allowRegex in allowRegexes)
|
||||
{
|
||||
if (!allowRegex.IsMatch(result.FullPath))
|
||||
{
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip) { continue; }
|
||||
foreach(var denyRegex in denyRegexes)
|
||||
{
|
||||
if (denyRegex.IsMatch(result.FullPath))
|
||||
{
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip) { continue; }
|
||||
var targetPath = Path.Combine(options.Output, result.FullPath);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
|
||||
using var fs = new FileStream(targetPath, FileMode.Create);
|
||||
result.Content.CopyTo(fs);
|
||||
if (options.PrintNames)
|
||||
{
|
||||
Console.WriteLine("Extracted {0}.", result.FullPath);
|
||||
}
|
||||
Logger.Trace("Extracted {0}", result.FullPath);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Logger.Fatal(e, "Failed to create file at {0}.", targetPath);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
private static NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,159 @@
|
|||
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
|
||||
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using NLog;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.CST.RecursiveExtractor.Cli;
|
||||
|
||||
namespace Microsoft.CST.RecursiveExtractor.Tests
|
||||
{
|
||||
[TestClass]
|
||||
public class ExtractorCliTests
|
||||
{
|
||||
[DataTestMethod]
|
||||
[DataRow("Shared.zip")]
|
||||
[DataRow("Shared.7z")]
|
||||
[DataRow("Shared.Tar")]
|
||||
[DataRow("Shared.rar")]
|
||||
[DataRow("Shared.rar4")]
|
||||
[DataRow("Shared.tar.bz2")]
|
||||
[DataRow("Shared.tar.gz")]
|
||||
[DataRow("Shared.tar.xz")]
|
||||
[DataRow("sysvbanner_1.0-17fakesync1_amd64.deb", 8)]
|
||||
[DataRow("Shared.a", 1)]
|
||||
[DataRow("Shared.deb", 27)]
|
||||
[DataRow("Shared.ar")]
|
||||
[DataRow("Shared.iso")]
|
||||
[DataRow("Shared.vhd", 29)] // 26 + Some invisible system files
|
||||
[DataRow("Shared.vhdx")]
|
||||
[DataRow("Shared.wim")]
|
||||
[DataRow("Empty.vmdk", 0)]
|
||||
[DataRow("TextFile.md", 1)]
|
||||
[DataRow("Nested.Zip", 26 * 8 + 1)] // there's one extra metadata file in there
|
||||
public void ExtractArchive(string fileName, int expectedNumFiles = 26)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
|
||||
RecursiveExtractorClient.ExtractCommand(new ExtractCommandOptions() { Input = path, Output = directory, Verbose = true });
|
||||
var files = Array.Empty<string>();
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
files = Directory.EnumerateFiles(directory, "*.*", SearchOption.AllDirectories).ToArray();
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
Assert.AreEqual(expectedNumFiles, files.Length);
|
||||
}
|
||||
|
||||
[DataTestMethod]
|
||||
[DataRow("Shared.zip")]
|
||||
[DataRow("Shared.7z")]
|
||||
[DataRow("Shared.Tar")]
|
||||
[DataRow("Shared.rar")]
|
||||
[DataRow("Shared.rar4")]
|
||||
[DataRow("Shared.tar.bz2")]
|
||||
[DataRow("Shared.tar.gz")]
|
||||
[DataRow("Shared.tar.xz")]
|
||||
[DataRow("sysvbanner_1.0-17fakesync1_amd64.deb", 0)]
|
||||
[DataRow("Shared.a", 0)]
|
||||
[DataRow("Shared.deb")]
|
||||
[DataRow("Shared.ar")]
|
||||
[DataRow("Shared.iso")]
|
||||
[DataRow("Shared.vhd")] // Filename formatting in the VHD has CS capitalized so nothing matches
|
||||
[DataRow("Shared.vhdx")]
|
||||
[DataRow("Shared.wim")]
|
||||
[DataRow("Empty.vmdk", 0)]
|
||||
[DataRow("TextFile.md", 0)]
|
||||
[DataRow("Nested.Zip", 22 * 8)]
|
||||
public void ExtractArchiveWithAllowFilters(string fileName, int expectedNumFiles = 22)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
|
||||
RecursiveExtractorClient.ExtractCommand(new ExtractCommandOptions()
|
||||
{
|
||||
Input = path,
|
||||
Output = directory,
|
||||
Verbose = true,
|
||||
AllowFilters = new string[]
|
||||
{
|
||||
".[cC][sS]$"
|
||||
}
|
||||
});
|
||||
var files = Array.Empty<string>();
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
files = Directory.EnumerateFiles(directory, "*.*", SearchOption.AllDirectories).ToArray();
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
Assert.AreEqual(expectedNumFiles, files.Length);
|
||||
}
|
||||
|
||||
[DataTestMethod]
|
||||
[DataRow("Shared.zip")]
|
||||
[DataRow("Shared.7z")]
|
||||
[DataRow("Shared.Tar")]
|
||||
[DataRow("Shared.rar")]
|
||||
[DataRow("Shared.rar4")]
|
||||
[DataRow("Shared.tar.bz2")]
|
||||
[DataRow("Shared.tar.gz")]
|
||||
[DataRow("Shared.tar.xz")]
|
||||
[DataRow("sysvbanner_1.0-17fakesync1_amd64.deb", 8)]
|
||||
[DataRow("Shared.a", 1)]
|
||||
[DataRow("Shared.deb", 5)]
|
||||
[DataRow("Shared.ar")]
|
||||
[DataRow("Shared.iso")]
|
||||
[DataRow("Shared.vhd", 7)] // 26 + Some invisible system files
|
||||
[DataRow("Shared.vhdx")]
|
||||
[DataRow("Shared.wim")]
|
||||
[DataRow("Empty.vmdk", 0)]
|
||||
[DataRow("TextFile.md", 1)]
|
||||
[DataRow("Nested.Zip", 4 * 8 + 1)] // there's one extra metadata file in there
|
||||
public void ExtractArchiveWithDenyFilters(string fileName, int expectedNumFiles = 4)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
|
||||
RecursiveExtractorClient.ExtractCommand(new ExtractCommandOptions()
|
||||
{
|
||||
Input = path,
|
||||
Output = directory,
|
||||
Verbose = true,
|
||||
DenyFilters = new string[]
|
||||
{
|
||||
".[cC][sS]$"
|
||||
}
|
||||
});
|
||||
var files = Array.Empty<string>();
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
files = Directory.EnumerateFiles(directory, "*.*", SearchOption.AllDirectories).ToArray();
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
Assert.AreEqual(expectedNumFiles, files.Length);
|
||||
}
|
||||
|
||||
[DataTestMethod]
|
||||
[DataRow("SharedEncrypted.7z")]
|
||||
[DataRow("SharedEncrypted.zip")]
|
||||
[DataRow("SharedEncrypted.rar4")]
|
||||
[DataRow("NestedEncrypted.7z", 26 * 3)] // there's one extra metadata file in there
|
||||
public void ExtractEncryptedArchive(string fileName, int expectedNumFiles = 26)
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
|
||||
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
|
||||
var passwords = ExtractorTests.TestArchivePasswords.Values.SelectMany(x => x);
|
||||
RecursiveExtractorClient.ExtractCommand(new ExtractCommandOptions() { Input = path, Output = directory, Verbose = true, Passwords = passwords });
|
||||
var files = Array.Empty<string>();
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
files = Directory.EnumerateFiles(directory, "*.*", SearchOption.AllDirectories).ToArray();
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
Assert.AreEqual(expectedNumFiles, files.Length);
|
||||
}
|
||||
|
||||
protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger();
|
||||
}
|
||||
}
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using NLog;
|
||||
using NLog.Config;
|
||||
using NLog.Targets;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
@ -42,7 +44,7 @@ namespace Microsoft.CST.RecursiveExtractor.Tests
|
|||
Assert.IsTrue(results.Count() == expectedNumFiles);
|
||||
}
|
||||
|
||||
private Dictionary<Regex, List<string>> TestArchivePasswords = new Dictionary<Regex, List<string>>()
|
||||
public static Dictionary<Regex, List<string>> TestArchivePasswords = new Dictionary<Regex, List<string>>()
|
||||
{
|
||||
{
|
||||
new Regex("\\.zip"),
|
||||
|
@ -57,8 +59,8 @@ namespace Microsoft.CST.RecursiveExtractor.Tests
|
|||
new List<string>()
|
||||
{
|
||||
"AnIncorrectPassword",
|
||||
"TheMagicWordIsTomato",
|
||||
"TheMagicWordIsLettuce"
|
||||
"TheMagicWordIsTomato", // SharedEncrypted.7z
|
||||
"TheMagicWordIsLettuce" // NestedEncrypted.7z
|
||||
}
|
||||
},
|
||||
{
|
||||
|
@ -241,7 +243,7 @@ namespace Microsoft.CST.RecursiveExtractor.Tests
|
|||
[DataRow("Shared.zip", ArchiveFileType.ZIP)]
|
||||
[DataRow("Shared.7z", ArchiveFileType.P7ZIP)]
|
||||
[DataRow("Shared.Tar", ArchiveFileType.TAR)]
|
||||
[DataRow("Shared.rar", ArchiveFileType.RAR)]
|
||||
[DataRow("Shared.rar", ArchiveFileType.RAR5)]
|
||||
[DataRow("Shared.rar4", ArchiveFileType.RAR)]
|
||||
[DataRow("Shared.tar.bz2", ArchiveFileType.BZIP2)]
|
||||
[DataRow("Shared.tar.gz", ArchiveFileType.GZIP)]
|
||||
|
@ -307,6 +309,31 @@ namespace Microsoft.CST.RecursiveExtractor.Tests
|
|||
Assert.Fail();
|
||||
}
|
||||
|
||||
[DataTestMethod]
|
||||
[DataRow("zip-slip-win.zip")]
|
||||
[DataRow("zip-slip-win.tar")]
|
||||
[DataRow("zip-slip.zip")]
|
||||
[DataRow("zip-slip.tar")]
|
||||
public void TestZipSlip(string fileName)
|
||||
{
|
||||
var extractor = new Extractor();
|
||||
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
|
||||
var results = extractor.ExtractFile(path, new ExtractorOptions()).ToList();
|
||||
Assert.IsTrue(results.All(x => !x.FullPath.Contains("..")));
|
||||
}
|
||||
[ClassInitialize]
|
||||
public static void ClassInitialize(TestContext context)
|
||||
{
|
||||
var config = new LoggingConfiguration();
|
||||
var consoleTarget = new ConsoleTarget
|
||||
{
|
||||
Name = "console",
|
||||
Layout = "${longdate}|${level:uppercase=true}|${logger}|${message}",
|
||||
};
|
||||
config.AddRule(LogLevel.Trace, LogLevel.Fatal, consoleTarget, "*");
|
||||
|
||||
LogManager.Configuration = config;
|
||||
}
|
||||
protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger();
|
||||
}
|
||||
}
|
|
@ -22,11 +22,13 @@
|
|||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="NLog.Extensions.Logging" Version="1.6.4" />
|
||||
<PackageReference Include="Sarif.Sdk" Version="2.2.5" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\RecursiveExtractor\RecursiveExtractor.csproj" />
|
||||
<ProjectReference Include="..\RecursiveExtractor.Cli\RecursiveExtractor.Cli.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="TestData\10GB.7z.bz2">
|
||||
|
@ -134,6 +136,18 @@
|
|||
<None Update="TestData\zbxl.zip">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="TestData\zip-slip-win.tar">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="TestData\zip-slip-win.zip">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="TestData\zip-slip.tar">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="TestData\zip-slip.zip">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Update="Nerdbank.GitVersioning" Version="3.1.91" />
|
||||
|
|
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
@ -15,6 +15,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RecursiveExtractor.Blazor", "RecursiveExtractor.Blazor\RecursiveExtractor.Blazor.csproj", "{18D0803C-052E-4338-9162-F2DB8F8E51E2}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RecursiveExtractor.Cli", "RecursiveExtractor.Cli\RecursiveExtractor.Cli.csproj", "{443B4E50-9AAF-436E-B3DF-644F782AF9B6}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
@ -33,6 +35,10 @@ Global
|
|||
{18D0803C-052E-4338-9162-F2DB8F8E51E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{18D0803C-052E-4338-9162-F2DB8F8E51E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{18D0803C-052E-4338-9162-F2DB8F8E51E2}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{443B4E50-9AAF-436E-B3DF-644F782AF9B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{443B4E50-9AAF-436E-B3DF-644F782AF9B6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{443B4E50-9AAF-436E-B3DF-644F782AF9B6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{443B4E50-9AAF-436E-B3DF-644F782AF9B6}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
|
@ -82,16 +82,6 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
var nameSpan = new byte[nameLength];
|
||||
// This should move us right to the file
|
||||
|
||||
/* Unmerged change from project 'RecursiveExtractor (netstandard2.1)'
|
||||
Before:
|
||||
fileEntry.Content.Read(nameSpan, 0, nameLength);
|
||||
|
||||
var entryStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
|
||||
After:
|
||||
fileEntry.Content.Read(nameSpan, 0, nameLength);
|
||||
|
||||
var entryStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
|
||||
*/
|
||||
fileEntry.Content.Read(nameSpan, 0, nameLength);
|
||||
|
||||
var entryStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
|
||||
|
@ -99,7 +89,7 @@ After:
|
|||
// The name length is included in the total size reported in the header
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, size - nameLength);
|
||||
|
||||
yield return new FileEntry(Encoding.ASCII.GetString(nameSpan), entryStream, fileEntry, true);
|
||||
yield return new FileEntry(Encoding.ASCII.GetString(nameSpan).TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
else if (filename.Equals('/'))
|
||||
|
@ -166,7 +156,7 @@ After:
|
|||
new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose) :
|
||||
(Stream)new MemoryStream((int)innerSize);
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, innerSize);
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
fileEntry.Content.Position = fileEntry.Content.Length - 1;
|
||||
|
@ -239,7 +229,7 @@ After:
|
|||
(Stream)new MemoryStream((int)innerSize);
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, innerSize);
|
||||
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
fileEntry.Content.Position = fileEntry.Content.Length - 1;
|
||||
|
@ -262,7 +252,7 @@ After:
|
|||
(Stream)new MemoryStream((int)size);
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, size);
|
||||
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -271,7 +261,7 @@ After:
|
|||
(Stream)new MemoryStream((int)size);
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, size);
|
||||
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -361,7 +351,7 @@ After:
|
|||
// The name length is included in the total size reported in the header
|
||||
await CopyStreamBytesAsync(fileEntry.Content, entryStream, size - nameLength);
|
||||
|
||||
yield return new FileEntry(Encoding.ASCII.GetString(nameSpan), entryStream, fileEntry, true);
|
||||
yield return new FileEntry(Encoding.ASCII.GetString(nameSpan).TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
else if (filename.Equals('/'))
|
||||
|
@ -428,7 +418,7 @@ After:
|
|||
new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose) :
|
||||
(Stream)new MemoryStream((int)innerSize);
|
||||
await CopyStreamBytesAsync(fileEntry.Content, entryStream, innerSize);
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
fileEntry.Content.Position = fileEntry.Content.Length - 1;
|
||||
|
@ -501,7 +491,7 @@ After:
|
|||
new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose) :
|
||||
(Stream)new MemoryStream((int)innerSize);
|
||||
await CopyStreamBytesAsync(fileEntry.Content, entryStream, innerSize);
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
}
|
||||
fileEntry.Content.Position = fileEntry.Content.Length - 1;
|
||||
|
@ -523,7 +513,7 @@ After:
|
|||
new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose) :
|
||||
(Stream)new MemoryStream((int)size);
|
||||
CopyStreamBytes(fileEntry.Content, entryStream, size);
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -531,7 +521,7 @@ After:
|
|||
new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose) :
|
||||
(Stream)new MemoryStream((int)size);
|
||||
await CopyStreamBytesAsync(fileEntry.Content, entryStream, size);
|
||||
yield return new FileEntry(filename, entryStream, fileEntry, true);
|
||||
yield return new FileEntry(filename.TrimEnd('/'), entryStream, fileEntry, true);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -40,6 +40,7 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
SetExtractor(ArchiveFileType.GZIP, new GzipExtractor(this));
|
||||
SetExtractor(ArchiveFileType.ISO_9660, new IsoExtractor(this));
|
||||
SetExtractor(ArchiveFileType.RAR, new RarExtractor(this));
|
||||
SetExtractor(ArchiveFileType.RAR5, new RarExtractor(this));
|
||||
SetExtractor(ArchiveFileType.P7ZIP, new SevenZipExtractor(this));
|
||||
SetExtractor(ArchiveFileType.TAR, new TarExtractor(this));
|
||||
SetExtractor(ArchiveFileType.VHD, new VhdExtractor(this));
|
||||
|
@ -53,6 +54,14 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
}
|
||||
}
|
||||
|
||||
public void Unset(ArchiveFileType targetType)
|
||||
{
|
||||
if (Extractors.ContainsKey(targetType))
|
||||
{
|
||||
Extractors.Remove(targetType);
|
||||
}
|
||||
}
|
||||
|
||||
public void SetExtractor(ArchiveFileType targetType, AsyncExtractorInterface implementation)
|
||||
{
|
||||
Extractors[targetType] = implementation;
|
||||
|
@ -229,8 +238,7 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
{
|
||||
directory = string.Empty;
|
||||
}
|
||||
// We give it a parent so we can give it a shortname. This is useful for Quine detection later.
|
||||
fileEntry = new FileEntry(file, stream, new FileEntry(directory, new MemoryStream()));
|
||||
fileEntry = new FileEntry(Path.GetFileName(file), stream);
|
||||
governor.ResetResourceGovernor(stream);
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
|
|
@ -44,9 +44,10 @@ namespace Microsoft.CST.RecursiveExtractor.Extractors
|
|||
foreach (var file in diskFiles)
|
||||
{
|
||||
Stream? fileStream = null;
|
||||
DiscFileInfo? fi = null;
|
||||
try
|
||||
{
|
||||
var fi = fs.GetFileInfo(file);
|
||||
fi = fs.GetFileInfo(file);
|
||||
governor.CheckResourceGovernor(fi.Length);
|
||||
fileStream = fi.OpenRead();
|
||||
}
|
||||
|
@ -54,9 +55,9 @@ namespace Microsoft.CST.RecursiveExtractor.Extractors
|
|||
{
|
||||
Logger.Debug(e, "Failed to open {0} in volume {1}", file, volume.Identity);
|
||||
}
|
||||
if (fileStream != null)
|
||||
if (fileStream != null && fi != null)
|
||||
{
|
||||
var newFileEntry = await FileEntry.FromStreamAsync($"{volume.Identity}{Path.DirectorySeparatorChar}{file}", fileStream, parent);
|
||||
var newFileEntry = await FileEntry.FromStreamAsync($"{volume.Identity}{Path.DirectorySeparatorChar}{fi.FullName}", fileStream, parent);
|
||||
var entries = Context.ExtractFileAsync(newFileEntry, options, governor);
|
||||
await foreach (var entry in entries)
|
||||
{
|
||||
|
@ -123,7 +124,7 @@ namespace Microsoft.CST.RecursiveExtractor.Extractors
|
|||
{
|
||||
if (file.Item2 != null)
|
||||
{
|
||||
var newFileEntry = new FileEntry($"{volume.Identity}\\{file.Item1.FullName}", file.Item2, parent);
|
||||
var newFileEntry = new FileEntry($"{volume.Identity}{Path.DirectorySeparatorChar}{file.Item1.FullName}", file.Item2, parent);
|
||||
var entries = Context.ExtractFile(newFileEntry, options, governor);
|
||||
files.PushRange(entries.ToArray());
|
||||
}
|
||||
|
|
|
@ -140,7 +140,9 @@ namespace Microsoft.CST.RecursiveExtractor.Extractors
|
|||
{
|
||||
try
|
||||
{
|
||||
var newFileEntry = new FileEntry(entry.entry.Key, entry.Item2, fileEntry);
|
||||
var name = entry.entry.Key.Replace('/', Path.DirectorySeparatorChar);
|
||||
|
||||
var newFileEntry = new FileEntry(name, entry.Item2, fileEntry);
|
||||
if (Extractor.IsQuine(newFileEntry))
|
||||
{
|
||||
Logger.Info(Extractor.IS_QUINE_STRING, fileEntry.Name, fileEntry.FullPath);
|
||||
|
|
|
@ -25,18 +25,31 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
public FileEntry(string name, Stream inputStream, FileEntry? parent = null, bool passthroughStream = false)
|
||||
{
|
||||
Parent = parent;
|
||||
Name = name;
|
||||
Passthrough = passthroughStream;
|
||||
|
||||
Name = Path.GetFileName(name);
|
||||
|
||||
if (parent == null)
|
||||
{
|
||||
ParentPath = null;
|
||||
FullPath = Name;
|
||||
FullPath = name;
|
||||
}
|
||||
else
|
||||
{
|
||||
ParentPath = parent.FullPath;
|
||||
FullPath = $"{ParentPath}{Path.DirectorySeparatorChar}{Name}";
|
||||
FullPath = $"{ParentPath}{Path.DirectorySeparatorChar}{name}";
|
||||
}
|
||||
var printPath = FullPath;
|
||||
if (FullPath.Contains(".."))
|
||||
{
|
||||
Logger.Info("ZipSlip detected in {0}. Removing unsafe path elements and extracting.", FullPath);
|
||||
// Replace .. for ZipSlip - https://snyk.io/research/zip-slip-vulnerability
|
||||
FullPath = FullPath.Replace("..", "");
|
||||
var doubleSeparator = $"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}";
|
||||
while (FullPath.Contains(doubleSeparator))
|
||||
{
|
||||
FullPath = FullPath.Replace(doubleSeparator, $"{Path.DirectorySeparatorChar}");
|
||||
}
|
||||
}
|
||||
|
||||
if (inputStream == null)
|
||||
|
@ -82,18 +95,18 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
{
|
||||
try
|
||||
{
|
||||
System.Threading.Tasks.Task.Run(() => inputStream.CopyToAsync(Content)).Wait();
|
||||
Task.Run(() => inputStream.CopyToAsync(Content)).Wait();
|
||||
}
|
||||
catch (Exception f)
|
||||
{
|
||||
var message = f.Message;
|
||||
var type = f.GetType();
|
||||
Logger.Debug("Failed to copy stream from {0} ({1}:{2})", FullPath, f.GetType(), f.Message);
|
||||
Logger.Debug("Failed to copy stream from {0} ({1}:{2})", printPath, f.GetType(), f.Message);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Logger.Debug("Failed to copy stream from {0} ({1}:{2})", FullPath, e.GetType(), e.Message);
|
||||
Logger.Debug("Failed to copy stream from {0} ({1}:{2})", printPath, e.GetType(), e.Message);
|
||||
}
|
||||
|
||||
if (inputStream.CanSeek && inputStream.Position != 0)
|
||||
|
@ -154,17 +167,19 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
{
|
||||
content = new MemoryStream();
|
||||
}
|
||||
string? ParentPath;
|
||||
|
||||
// Used for Debug statements
|
||||
string FullPath;
|
||||
|
||||
if (parent == null)
|
||||
{
|
||||
FullPath = name;
|
||||
}
|
||||
else
|
||||
{
|
||||
ParentPath = parent.FullPath;
|
||||
FullPath = $"{ParentPath}{Path.DirectorySeparatorChar}{name}";
|
||||
FullPath = $"{parent?.FullPath}{Path.DirectorySeparatorChar}{name}";
|
||||
}
|
||||
|
||||
// Back with a temporary filestream, this is optimized to be cached in memory when possible
|
||||
// automatically by .NET
|
||||
var Content = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
|
||||
|
|
|
@ -20,6 +20,7 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
GZIP,
|
||||
BZIP2,
|
||||
RAR,
|
||||
RAR5,
|
||||
P7ZIP,
|
||||
DEB,
|
||||
AR,
|
||||
|
@ -85,11 +86,14 @@ namespace Microsoft.CST.RecursiveExtractor
|
|||
{
|
||||
return ArchiveFileType.BZIP2;
|
||||
}
|
||||
if ((buffer[0] == 0x52 && buffer[1] == 0x61 && buffer[2] == 0x72 && buffer[3] == 0x21 && buffer[4] == 0x1A && buffer[5] == 0x07 && buffer[6] == 0x00) ||
|
||||
(buffer[0] == 0x52 && buffer[1] == 0x61 && buffer[2] == 0x72 && buffer[3] == 0x21 && buffer[4] == 0x1A && buffer[5] == 0x07 && buffer[6] == 0x01 && buffer[7] == 0x00))
|
||||
if (buffer[0] == 0x52 && buffer[1] == 0x61 && buffer[2] == 0x72 && buffer[3] == 0x21 && buffer[4] == 0x1A && buffer[5] == 0x07 && buffer[6] == 0x00)
|
||||
{
|
||||
return ArchiveFileType.RAR;
|
||||
}
|
||||
if (buffer[0] == 0x52 && buffer[1] == 0x61 && buffer[2] == 0x72 && buffer[3] == 0x21 && buffer[4] == 0x1A && buffer[5] == 0x07 && buffer[6] == 0x01 && buffer[7] == 0x00)
|
||||
{
|
||||
return ArchiveFileType.RAR5;
|
||||
}
|
||||
if (buffer[0] == 0x37 && buffer[1] == 0x7A && buffer[2] == 0xBC && buffer[3] == 0xAF && buffer[4] == 0x27 && buffer[5] == 0x1C)
|
||||
{
|
||||
return ArchiveFileType.P7ZIP;
|
||||
|
|
Загрузка…
Ссылка в новой задаче