diff --git a/build.proj b/build.proj index 154efbec16..498e0da6fb 100644 --- a/build.proj +++ b/build.proj @@ -173,7 +173,7 @@ Microsoft.Powershell.*.dll,System*.dll,Microsoft.VisualBasic.dll,Microsoft.CSharp.dll,Microsoft.CodeAnalysis.dll,Microsoft.CodeAnalysis.CSharp.dll - System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll,System.Management.dll + System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll,System.Management.dll,System.Text.Json.dll,System.Threading.Tasks.Extensions.dll diff --git a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs index 1348d79725..25d6621cde 100644 --- a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs +++ b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs @@ -83,5 +83,12 @@ namespace Microsoft.Azure.Commands.Management.Storage.Test.ScenarioTests { TestController.NewInstance.RunPsTest(_logger, $"Test-Common -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\""); } + + [Fact] + [Trait(Category.RunType, Category.LiveOnly)] + public void TestDatalakeGen2() + { + TestController.NewInstance.RunPsTest(_logger, $"Test-DatalakeGen2 -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\""); + } } } diff --git a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 index a970ac2b0f..2ab1e927e6 100644 --- a/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 +++ b/src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1 @@ -207,6 +207,26 @@ function Test-Blob Assert-AreEqual $blob.Count 2 Get-AzStorageBlob -Container $containerName -Blob $objectName2 -Context $storageContext | Remove-AzStorageBlob -Force + #check XSCL Track2 Items works for container + $container = Get-AzStorageContainer $containerName -Context $storageContext + $containerProperties = $container.BlobContainerClient.GetProperties().Value + Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag + Set-AzStorageContainerAcl $containerName -Context $storageContext -Permission Blob + $containerProperties = $container.BlobContainerClient.GetProperties().Value + Assert-AreNotEqual $container.BlobContainerProperties.ETag $containerProperties.ETag + $container.FetchAttributes() + Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag + + #check XSCL Track2 Items works for Blob + $blob = Get-AzStorageBlob -Container $containerName -Blob $objectName1 -Context $storageContext + $blobProperties = $blob.BlobClient.GetProperties().Value + Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag + Set-AzStorageBlobContent -File $localSrcFile -Container $containerName -Blob $objectName1 -Force -Context $storageContext + $blobProperties = $blob.BlobClient.GetProperties().Value + Assert-AreNotEqual $blob.BlobProperties.ETag $blobProperties.ETag + $blob.FetchAttributes() + Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag + # Copy blob to the same container, but with a different name. Start-AzStorageBlobCopy -srcContainer $containerName -SrcBlob $objectName1 -DestContainer $containerName -DestBlob $objectName2 -StandardBlobTier $StandardBlobTier -RehydratePriority High -Context $storageContext -DestContext $storageContext Get-AzStorageBlobCopyState -Container $containerName -Blob $objectName2 -Context $storageContext @@ -684,6 +704,153 @@ function Test-Common } } +<# + .SYNOPSIS + Tests DatalakeGen-only related commands. +#> +function Test-DatalakeGen2 +{ + Param( + [Parameter(Mandatory = $True)] + [string] + $StorageAccountName, + [Parameter(Mandatory = $True)] + [string] + $ResourceGroupName + ) + + New-TestResourceGroupAndStorageAccount -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -EnableHNFS $true + + try{ + + $storageAccountKeyValue = $(Get-AzStorageAccountKey -ResourceGroupName $ResourceGroupName -Name $StorageAccountName)[0].Value + $storageContext = New-AzStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $storageAccountKeyValue + + $localSrcFile = "localsrcDatalakeGen2testfile.psd1" #The file need exist before test, and should be 512 bytes aligned + New-Item $localSrcFile -ItemType File -Force + $localDestFile = "localdestDatalakeGen2testfile.txt" + + $filesystemName = "adlsgen2testfilesystem" + $directoryPath1 = "dir1" + $directoryPath2 = "dir2" + $directoryPath3 = "dir3" + $filePath1 = "dir1/Item1.txt" + $filePath2 = "dir2/Item2.txt" + $filePath3 = "dir2/Item3.txt" + $ContentType = "image/jpeg" + $ContentMD5 = "i727sP7HigloQDsqadNLHw==" + + # Create FileSystem (actually a container) + New-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext + + # Create folders + $dir1 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Directory -Permission rwxrwxrwx -Umask ---rwx--- -Property @{"ContentEncoding" = "UDF8"; "CacheControl" = "READ"} -Metadata @{"tag1" = "value1"; "tag2" = "value2" } + Assert-AreEqual $dir1.Path $directoryPath1 + Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rwx---rwx" + $dir2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath2 -Directory + + # Create (upload) File + $t = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Source $localSrcFile -Force -AsJob + $t | wait-job + Assert-AreEqual $t.State "Completed" + Assert-AreEqual $t.Error $null + $file2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -Source $localSrcFile -Permission rwxrwxrwx -Umask ---rwx--- -Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5} -Metadata @{"tag1" = "value1"; "tag2" = "value2" } + Assert-AreEqual $file2.Path $filePath2 + Assert-AreEqual $file2.Properties.ContentType $ContentType + Assert-AreEqual $file2.Properties.Metadata.Count 2 + Assert-AreEqual $file2.Permissions.ToSymbolicPermissions() "rwx---rwx" + + # update Blob and Directory + $ContentType = "application/octet-stream" + $ContentMD5 = "NW/H9Zxr2md6L1/yhNKdew==" + $ContentEncoding = "UDF8" + ## create ACL with 3 ACEs + $acl = New-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rw- + $acl = New-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl + $acl = New-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "-wx" -InputObject $acl + ##Update File with pipeline + $file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 | Update-AzDataLakeGen2Item ` + -Acl $acl ` + -Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5} ` + -Metadata @{"tag1" = "value1"; "tag2" = "value2" } ` + -Permission rw-rw--wx ` + -Owner '$superuser' ` + -Group '$superuser' + $file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 + Assert-AreEqual $file1.Path $filePath1 + Assert-AreEqual $file1.Permissions.ToSymbolicPermissions() "rw-rw--wx" + Assert-AreEqual $file1.Properties.ContentType $ContentType + Assert-AreEqual $file1.Properties.Metadata.Count 2 + Assert-AreEqual $file1.Owner '$superuser' + Assert-AreEqual $file1.Group '$superuser' + ## Update Directory + $dir1 = Update-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 ` + -Acl $acl ` + -Property @{"ContentEncoding" = $ContentEncoding} ` + -Metadata @{"tag1" = "value1"; "tag2" = "value2" } ` + -Permission rw-rw--wx ` + -Owner '$superuser' ` + -Group '$superuser' + $dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 + Assert-AreEqual $dir1.Path $directoryPath1 + Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rw-rw--wx" + Assert-AreEqual $dir1.Properties.ContentEncoding $ContentEncoding + Assert-AreEqual $dir1.Properties.Metadata.Count 3 # inlucde "hdi_isfolder" which is handle by server + Assert-AreEqual $dir1.Owner '$superuser' + Assert-AreEqual $dir1.Group '$superuser' + + #list Items + ## List direct Items from FileSystem + $items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -FetchPermission + Assert-AreEqual $items.Count 2 + Assert-NotNull $items[0].Permissions + $items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -Recurse + Assert-AreEqual $items.Count 4 + Assert-Null $items[0].Permissions + + #download File + $t = Get-AzDataLakeGen2ItemContent -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Destination $localDestFile -AsJob -Force + $t | Wait-Job + Assert-AreEqual $t.State "Completed" + Assert-AreEqual $t.Error $null + Assert-AreEqual (Get-FileHash -Path $localDestFile -Algorithm MD5).Hash (Get-FileHash -Path $localSrcFile -Algorithm MD5).Hash + + # Move Items + ## Move File + $file3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -DestFileSystem $filesystemName -DestPath $filePath3 -Force + $file3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath3 + Assert-AreEqual $file3.Path $filePath3 + Assert-AreEqual $file3.Permissions $file2.Permissions + $file2 = $file3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $filePath2 + $file2 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 + Assert-AreEqual $file2.Path $filePath2 + Assert-AreEqual $file2.Permissions $file3.Permissions + ## Move Folder + $dir3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -DestFileSystem $filesystemName -DestPath $directoryPath3 + $dir3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath3 + Assert-AreEqual $dir3.Path $directoryPath3 + Assert-AreEqual $dir3.Permissions $dir1.Permissions + $dir1 = $dir3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $directoryPath1 + $dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 + Assert-AreEqual $dir1.Path $directoryPath1 + + # Remove Items + Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Force + Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Force + + # Clean Storage Account + Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName | Remove-AzDataLakeGen2Item -Force + + # remove File system (actually a container) + Remove-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext + + } + finally + { + Clean-ResourceGroup $ResourceGroupName + } +} + function New-TestResourceGroupAndStorageAccount { Param( diff --git a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj index 5dc2f86913..14731423dd 100644 --- a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj +++ b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj @@ -11,6 +11,8 @@ + + diff --git a/src/Storage/Storage.Management/Az.Storage.psd1 b/src/Storage/Storage.Management/Az.Storage.psd1 index ec7a46f367..078cc51ec9 100644 --- a/src/Storage/Storage.Management/Az.Storage.psd1 +++ b/src/Storage/Storage.Management/Az.Storage.psd1 @@ -66,7 +66,8 @@ RequiredAssemblies = 'Microsoft.Azure.Management.Storage.dll', 'Microsoft.Azure.Storage.DataMovement.dll', 'Microsoft.Azure.DocumentDB.Core.dll', 'Microsoft.OData.Core.dll', 'Microsoft.OData.Edm.dll', 'Microsoft.Spatial.dll', - 'Microsoft.Azure.KeyVault.Core.dll' + 'Microsoft.Azure.KeyVault.Core.dll','Azure.Storage.Blobs.dll', + 'Azure.Storage.Common.dll', 'Azure.Storage.Files.DataLake.dll' # Script files (.ps1) that are run in the caller's environment prior to importing this module. # ScriptsToProcess = @() @@ -158,7 +159,12 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey', 'Revoke-AzStorageAccountUserDelegationKeys', 'Get-AzStorageFileHandle', 'Close-AzStorageFileHandle', 'New-AzRmStorageShare', 'Remove-AzRmStorageShare', - 'Get-AzRmStorageShare', 'Update-AzRmStorageShare' + 'Get-AzRmStorageShare', 'Update-AzRmStorageShare', + 'Get-AzDataLakeGen2ChildItem', 'Get-AzDataLakeGen2Item', + 'New-AzDataLakeGen2Item', 'Move-AzDataLakeGen2Item', + 'Remove-AzDataLakeGen2Item', 'Update-AzDataLakeGen2Item', + 'Set-AzDataLakeGen2ItemAclObject', 'Get-AzDataLakeGen2ItemContent' + # Variables to export from this module # VariablesToExport = @() @@ -166,7 +172,9 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey', # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. AliasesToExport = 'Get-AzStorageContainerAcl', 'Start-CopyAzureStorageBlob', 'Stop-CopyAzureStorageBlob', 'Enable-AzStorageSoftDelete', - 'Disable-AzStorageSoftDelete' + 'Disable-AzStorageSoftDelete', + 'New-AzDatalakeGen2FileSystem', 'Remove-AzDatalakeGen2FileSystem', + 'Get-AzDatalakeGen2FileSystem', 'New-AzDataLakeGen2ItemAclObject' # DSC resources to export from this module # DscResourcesToExport = @() diff --git a/src/Storage/Storage.Management/ChangeLog.md b/src/Storage/Storage.Management/ChangeLog.md index 8f7e29cec1..afc99223e3 100644 --- a/src/Storage/Storage.Management/ChangeLog.md +++ b/src/Storage/Storage.Management/ChangeLog.md @@ -22,6 +22,15 @@ * Supported new SkuName StandardGZRS, StandardRAGZRS when create/update Storage account - `New-AzStorageAccount` - `Set-AzStorageAccount` +* Support DataLake Gen2 + - New-AzDataLakeGen2Item + - Get-AzDataLakeGen2Item + - Get-AzDataLakeGen2ChildItem + - Move-AzDataLakeGen2Item + - Set-AzDataLakeGen2ItemAclObject + - Update-AzDataLakeGen2Item + - Get-AzDataLakeGen2ItemContent + - Remove-AzDataLakeGen2Item ## Version 1.13.0 * Supported AllowProtectedAppendWrite in ImmutabilityPolicy diff --git a/src/Storage/Storage.Management/Storage.Management.csproj b/src/Storage/Storage.Management/Storage.Management.csproj index 1118e44357..4837961d56 100644 --- a/src/Storage/Storage.Management/Storage.Management.csproj +++ b/src/Storage/Storage.Management/Storage.Management.csproj @@ -15,11 +15,12 @@ - - - + + + - + + diff --git a/src/Storage/Storage.Management/help/Az.Storage.md b/src/Storage/Storage.Management/help/Az.Storage.md index 700d2ce81f..622fffaba5 100644 --- a/src/Storage/Storage.Management/help/Az.Storage.md +++ b/src/Storage/Storage.Management/help/Az.Storage.md @@ -41,6 +41,15 @@ Enable delete retention policy for the Azure Storage Blob service. ### [Enable-AzStorageStaticWebsite](Enable-AzStorageStaticWebsite.md) Enable static website for the Azure Storage account. +### [Get-AzDataLakeGen2ChildItem](Get-AzDataLakeGen2ChildItem.md) +Lists sub directorys and files from a directory or filesystem root. + +### [Get-AzDataLakeGen2Item](Get-AzDataLakeGen2Item.md) +Gets the details of a file or directory in a filesystem. + +### [Get-AzDataLakeGen2ItemContent](Get-AzDataLakeGen2ItemContent.md) +Download a file. + ### [Get-AzRmStorageContainer](Get-AzRmStorageContainer.md) Gets or lists Storage blob containers @@ -131,6 +140,12 @@ Gets the Storage resource usage of the current subscription. ### [Lock-AzRmStorageContainerImmutabilityPolicy](Lock-AzRmStorageContainerImmutabilityPolicy.md) Locks ImmutabilityPolicy of a Storage blob containers +### [Move-AzDataLakeGen2Item](Move-AzDataLakeGen2Item.md) +Move a file or directory to another a file or directory in same Storage account. + +### [New-AzDataLakeGen2Item](New-AzDataLakeGen2Item.md) +Create a file or directory in a filesystem. + ### [New-AzRmStorageContainer](New-AzRmStorageContainer.md) Creates a Storage blob container @@ -200,6 +215,9 @@ Generates an SAS token for an Azure Storage table. ### [New-AzStorageTableStoredAccessPolicy](New-AzStorageTableStoredAccessPolicy.md) Creates a stored access policy for an Azure storage table. +### [Remove-AzDataLakeGen2Item](Remove-AzDataLakeGen2Item.md) +Remove a file or directory. + ### [Remove-AzRmStorageContainer](Remove-AzRmStorageContainer.md) Removes a Storage blob container @@ -263,6 +281,9 @@ Revoke all User Delegation keys of a Storage account. ### [Set-AzCurrentStorageAccount](Set-AzCurrentStorageAccount.md) Modifies the current Storage account of the specified subscription. +### [Set-AzDataLakeGen2ItemAclObject](Set-AzDataLakeGen2ItemAclObject.md) +Creates/Updates a DataLake gen2 item ACL object, which can be used in Update-AzDataLakeGen2Item cmdlet. + ### [Set-AzRmStorageContainerImmutabilityPolicy](Set-AzRmStorageContainerImmutabilityPolicy.md) Creates or updates ImmutabilityPolicy of a Storage blob containers @@ -320,6 +341,9 @@ Stops a copy operation. ### [Stop-AzStorageFileCopy](Stop-AzStorageFileCopy.md) Stops a copy operation to the specified destination file. +### [Update-AzDataLakeGen2Item](Update-AzDataLakeGen2Item.md) +Update a file or directory on properties, metadata, permission, ACL, and owner. + ### [Update-AzRmStorageContainer](Update-AzRmStorageContainer.md) Modifies a Storage blob container diff --git a/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ChildItem.md b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ChildItem.md new file mode 100644 index 0000000000..107c7d5b83 --- /dev/null +++ b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ChildItem.md @@ -0,0 +1,249 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/get-azdatalakegen2childitem +schema: 2.0.0 +--- + +# Get-AzDataLakeGen2ChildItem + +## SYNOPSIS +Lists sub directorys and files from a directory or filesystem root. + +## SYNTAX + +``` +Get-AzDataLakeGen2ChildItem [-FileSystem] [[-Path] ] [-FetchProperty] [-Recurse] + [-MaxCount ] [-ContinuationToken ] [-AsJob] [-OutputUserPrincipalName] + [-Context ] [-DefaultProfile ] [] +``` + +## DESCRIPTION +The **Get-AzDataLakeGen2ChildItem** cmdlet lists sub directorys and files in a directory or Filesystem in an Azure storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: List the direct sub items from a Filesystem +``` +PS C:\>Get-AzDataLakeGen2ChildItem -FileSystem "filesystem1" + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1 True 2020-03-13 13:07:34Z rwxr-x--- $superuser $superuser +dir2 True 2020-03-23 09:28:36Z rwxr-x--- $superuser $superuser +``` + +This command lists the direct sub items from a Filesystem + +### Example 2: List recursively from a directory, and fetch Properties/ACL +``` +PS C:\>Get-AzDataLakeGen2ChildItem -FileSystem "filesystem1" -Path "dir1/" -Recurse -FetchProperty + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/dir3 True 2020-03-23 09:34:31Z rwx---rwx $superuser $superuser +dir1/file1 False 1024 2020-03-23 09:29:18Z rwx---rwx $superuser $superuser +dir1/testfile_1K_0 False 1024 2020-03-23 09:29:21Z rw-r----- $superuser $superuser +``` + +This command lists the direct sub items from a Filesystem + +### Example 3: List items recursively from a Filesystem in multiple batches +``` +PS C:\> $MaxReturn = 10000 +PS C:\> $FileSystemName = "filesystem1" +PS C:\> $Total = 0 +PS C:\> $Token = $Null +PS C:\> do + { + $items = Get-AzDataLakeGen2ChildItem -FileSystem $FileSystemName -Recurse -MaxCount $MaxReturn -ContinuationToken $Token + $Total += $items.Count + if($items.Length -le 0) { Break;} + $Token = $items[$items.Count -1].ContinuationToken; + } + While ($Token -ne $Null) +PS C:\> Echo "Total $Total items in Filesystem $FileSystemName" +``` + +This example uses the *MaxCount* and *ContinuationToken* parameters to list items recursively from a Filesystem in multiple batches. +The first four commands assign values to variables to use in the example. +The fifth command specifies a **Do-While** statement that uses the **Get-AzDataLakeGen2ChildItem** cmdlet to list items. +The statement includes the continuation token stored in the $Token variable. +$Token changes value as the loop runs. +The final command uses the **Echo** command to display the total. + +## PARAMETERS + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -ContinuationToken +Continuation Token. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FetchProperty +Fetch the datalake item properties and ACL. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: FetchPermission + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -MaxCount +The max count of the blobs that can return. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -OutputUserPrincipalName +If speicify this parameter, the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. If not speicify this parameter, the values will be returned as Azure Active Directory Object IDs. Note that group and application Object IDs are not translated because they do not have unique friendly names. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: UserPrincipalName + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be retrieved. +Should be a directory, in the format 'directory1/directory2/'. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: 1 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Recurse +Indicates if will recursively get the Child Item. +The default is false. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Get-AzDataLakeGen2Item.md b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2Item.md new file mode 100644 index 0000000000..83f6980771 --- /dev/null +++ b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2Item.md @@ -0,0 +1,186 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/get-azdatalakegen2item +schema: 2.0.0 +--- + +# Get-AzDataLakeGen2Item + +## SYNOPSIS +Gets the details of a file or directory in a filesystem. + +## SYNTAX + +``` +Get-AzDataLakeGen2Item [-FileSystem] [-Path ] [-Context ] + [-DefaultProfile ] [] +``` + +## DESCRIPTION +The **Get-AzDataLakeGen2Item** cmdlet gets the details of a file or directory in a Filesystem in an Azure storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Get a directory from a Filesystem, and show the details +``` +PS C:\> $dir1 = Get-AzDataLakeGen2tem -FileSystem "filesystem1" -Path "dir1/" +PS C:\> $dir1 + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1 True 2020-03-23 09:15:56Z rwx---rwx $superuser $superuser + +PS C:\WINDOWS\system32> $dir1.ACL + +DefaultScope AccessControlType EntityId Permissions +------------ ----------------- -------- ----------- +False User rwx +False Group --- +False Other rwx + +PS C:\WINDOWS\system32> $dir1.Permissions + +Owner : Execute, Write, Read +Group : None +Other : Execute, Write, Read +StickyBit : False +ExtendedAcls : False + +PS C:\WINDOWS\system32> $dir1.Properties.Metadata + +Key Value +--- ----- +hdi_isfolder true +tag1 value1 +tag2 value2 + +PS C:\WINDOWS\system32> $dir1.Properties + +LastModified : 3/23/2020 9:15:56 AM +00:00 +CreatedOn : 3/23/2020 9:15:56 AM +00:00 +Metadata : {[hdi_isfolder, true], [tag1, value1], [tag2, value2]} +CopyCompletedOn : 1/1/0001 12:00:00 AM +00:00 +CopyStatusDescription : +CopyId : +CopyProgress : +CopySource : +CopyStatus : Pending +IsIncrementalCopy : False +LeaseDuration : Infinite +LeaseState : Available +LeaseStatus : Unlocked +ContentLength : 0 +ContentType : application/octet-stream +ETag : "0x8D7CF0ACBA35FA8" +ContentHash : +ContentEncoding : UDF12 +ContentDisposition : +ContentLanguage : +CacheControl : READ +AcceptRanges : bytes +IsServerEncrypted : True +EncryptionKeySha256 : +AccessTier : Cool +ArchiveStatus : +AccessTierChangedOn : 1/1/0001 12:00:00 AM +00:00 +``` + +This command gets a directory from a Filesystem, and show the details. + +### Example 2: Get a file from a Filesystem +``` +PS C:\> Get-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/file1" + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/file1 False 1024 2020-03-23 09:20:37Z rwx---rwx $superuser $superuser +``` + +This command gets the details of a file from a Filesystem. + +## PARAMETERS + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be retrieved. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/'. +Not specify this parameter to get the root directory of the Filesystem. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ItemContent.md b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ItemContent.md new file mode 100644 index 0000000000..22331672e5 --- /dev/null +++ b/src/Storage/Storage.Management/help/Get-AzDataLakeGen2ItemContent.md @@ -0,0 +1,263 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/get-azdatalakegen2itemcontent +schema: 2.0.0 +--- + +# Get-AzDataLakeGen2ItemContent + +## SYNOPSIS +Download a file. + +## SYNTAX + +### ReceiveManual (Default) +``` +Get-AzDataLakeGen2ItemContent [-FileSystem] [-Path] [-Destination ] [-CheckMd5] + [-Force] [-AsJob] [-Context ] [-DefaultProfile ] + [-ConcurrentTaskCount ] [-WhatIf] [-Confirm] [] +``` + +### ItemPipeline +``` +Get-AzDataLakeGen2ItemContent -InputObject [-Destination ] [-CheckMd5] [-Force] + [-AsJob] [-Context ] [-DefaultProfile ] + [-ConcurrentTaskCount ] [-WhatIf] [-Confirm] [] +``` + +## DESCRIPTION +The **Get-AzDataLakeGen2ItemContent** cmdlet download a file in a Filesystem in an Azure storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Download a file without prompt +``` +PS C:\> Get-AzDataLakeGen2ItemContent -FileSystem "filesystem1" -Path "dir1/file1" -Destination $localDestFile -Force + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/file1 False 1024 2020-03-23 09:29:18Z rwx---rwx $superuser $superuser +``` + +This command downloads a file to a local file without prompt. + +### Example 2: Get a file, then pipeline to download the file to a local file +``` +PS C:\> Get-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/file1" | Get-AzDataLakeGen2ItemContent -Destination $localDestFile + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/file1 False 1024 2020-03-23 09:29:18Z rwx---rwx $superuser $superuser +``` + +This command first gets a file, then pipeline to download the file to a local file. + +## PARAMETERS + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -CheckMd5 +check the md5sum + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ConcurrentTaskCount +The total amount of concurrent async tasks. +The default value is 10. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Destination +Destination local file path. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Force +Force to overwrite the existing blob or file + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +Azure Datalake Gen2 Item Object to download. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item +Parameter Sets: ItemPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be removed. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/' + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Move-AzDataLakeGen2Item.md b/src/Storage/Storage.Management/help/Move-AzDataLakeGen2Item.md new file mode 100644 index 0000000000..2e97f6a000 --- /dev/null +++ b/src/Storage/Storage.Management/help/Move-AzDataLakeGen2Item.md @@ -0,0 +1,232 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/move-azdatalakegen2item +schema: 2.0.0 +--- + +# Move-AzDataLakeGen2Item + +## SYNOPSIS +Move a file or directory to another a file or directory in same Storage account. + +## SYNTAX + +### ReceiveManual (Default) +``` +Move-AzDataLakeGen2Item [-FileSystem] [-Path] -DestFileSystem -DestPath + [-Force] [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +### ItemPipeline +``` +Move-AzDataLakeGen2Item -InputObject -DestFileSystem -DestPath + [-Force] [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +The **Move-AzDataLakeGen2Item** cmdlet moves a a file or directory to another a file or directory in same Storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Move a fold in same Filesystem +``` +PS C:\> Move-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/" -DestFileSystem "filesystem1" -DestPath "dir3/" + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir3 True 2020-03-13 13:07:34Z rwxrw-rw- $superuser $superuser +``` + +This command move directory 'dir1' to directory 'dir3' in the same Filesystem. + +### Example 2: Move a file by pipeline, to another Filesystem in the same Storage account without prompt +``` +PS C:\> Get-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/file1" | Move-AzDataLakeGen2Item -DestFileSystem "filesystem2" -DestPath "dir2/file2" -Force + + FileSystem Name: filesystem2 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir2/file2 False 1024 2020-03-23 09:57:33Z rwxrw-rw- $superuser $superuser +``` + +This command move file 'dir1/file1' in 'filesystem1' to file 'dir2/file2' in 'filesystem2' in the same Storage account without prompt. + +## PARAMETERS + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestFileSystem +Dest FileSystem name + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DestPath +Dest Blob path + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Force +Force to over write the destination. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +Azure Datalake Gen2 Item Object to move from. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item +Parameter Sets: ItemPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be move from. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/' + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/New-AzDataLakeGen2Item.md b/src/Storage/Storage.Management/help/New-AzDataLakeGen2Item.md new file mode 100644 index 0000000000..86a6ca1530 --- /dev/null +++ b/src/Storage/Storage.Management/help/New-AzDataLakeGen2Item.md @@ -0,0 +1,315 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/new-azdatalakegen2item +schema: 2.0.0 +--- + +# New-AzDataLakeGen2Item + +## SYNOPSIS +Create a file or directory in a filesystem. + +## SYNTAX + +### File (Default) +``` +New-AzDataLakeGen2Item [-FileSystem] [-Path] -Source [-Umask ] + [-Permission ] [-Property ] [-Metadata ] [-Force] [-AsJob] + [-Context ] [-DefaultProfile ] [-ConcurrentTaskCount ] + [-WhatIf] [-Confirm] [] +``` + +### Directory +``` +New-AzDataLakeGen2Item [-FileSystem] [-Path] [-Directory] [-Umask ] + [-Permission ] [-Property ] [-Metadata ] [-Force] [-AsJob] + [-Context ] [-DefaultProfile ] [-ConcurrentTaskCount ] + [-WhatIf] [-Confirm] [] +``` + +## DESCRIPTION +The **New-AzDataLakeGen2Item** cmdlet creates a file or directory in a Filesystem in an Azure storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Create a directory with specified permission, Umask, properties, and metadata +``` +PS C:\>New-AzDataLakeGen2Item -FileSystem "testfilesystem" -Path "dir1/dir2/" -Directory -Permission rwxrwxrwx -Umask ---rw---- -Property @{"CacheControl" = "READ"; "ContentDisposition" = "True"} -Metadata @{"tag1" = "value1"; "tag2" = "value2" } + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/dir2 True 2020-03-23 09:15:56Z rwx---rwx $superuser $superuser +``` + +This command creates a directory with specified Permission, Umask, properties, and metadata + +### Example 2: Create(upload) a data lake file from a local source file, and the cmdlet runs in background +``` +PS C:\> $task = New-AzDataLakeGen2Item -FileSystem "testfilesystem" -Path "dir1/dir2/file1" -Source "c:\sourcefile.txt" -Force -asjob +PS C:\> $task | Wait-Job +PS C:\> $task.Output + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/dir2/file1 False 14400000 2020-03-23 09:19:13Z rw-r----- $superuser $superuser +``` + +This command creates(upload) a data lake file from a local source file, and the cmdlet runs in background. + +## PARAMETERS + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ConcurrentTaskCount +The total amount of concurrent async tasks. The default value is 10. + +```yaml +Type: System.Nullable`1[System.Int32] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Directory +Indicates that this new item is a directory and not a file. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: Directory +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Force +If passed then new item is created without any prompt + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Metadata +Specifies metadata for the created directory or file. + +```yaml +Type: System.Collections.Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be create. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/' + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Permission +Sets POSIX access permissions for the file owner, the file owning group, and others. +Each class may be granted read, write, or execute permission. +Symbolic (rwxrw-rw-) is supported. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Property +Specifies properties for the created directory or file. +The supported properties for file are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentMD5, ContentType. +The supported properties for directory are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage. + +```yaml +Type: System.Collections.Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Source +Specify the local source file path which will be upload to a Datalake Gen2 file. + +```yaml +Type: System.String +Parameter Sets: File +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Umask +When creating New Item and the parent directory does not have a default ACL, the umask restricts the permissions of the file or directory to be created. +The resulting permission is given by p & ^u, where p is the permission and u is the umask. +Symbolic (rwxrw-rw-) is supported. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Remove-AzDataLakeGen2Item.md b/src/Storage/Storage.Management/help/Remove-AzDataLakeGen2Item.md new file mode 100644 index 0000000000..28c226931c --- /dev/null +++ b/src/Storage/Storage.Management/help/Remove-AzDataLakeGen2Item.md @@ -0,0 +1,227 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/remove-azdatalakegen2item +schema: 2.0.0 +--- + +# Remove-AzDataLakeGen2Item + +## SYNOPSIS +Remove a file or directory. + +## SYNTAX + +### ReceiveManual (Default) +``` +Remove-AzDataLakeGen2Item [-FileSystem] [-Path] [-Force] [-AsJob] [-PassThru] + [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +### ItemPipeline +``` +Remove-AzDataLakeGen2Item -InputObject [-Force] [-AsJob] [-PassThru] + [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +The **Remove-AzDataLakeGen2Item** cmdlet removes a file or directory from a Storage account. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Removes a directory +``` +PS C:\>Remove-AzDataLakeGen2tem -FileSystem "filesystem1" -Path "dir1/" +``` + +This command removes a directory from a Filesystem. + +### Example 2: Removes a file without prompt +``` +PS C:\>Remove-AzDataLakeGen2tem -FileSystem "filesystem1" -Path "dir1/file1" -Force +``` + +This command removes a directory from a Filesystem, without prompt. + +### Example 3: Remove all items in a Filesystem with pipeline +``` +PS C:\>Get-AzDataLakeGen2ChildItem -FileSystem "filesystem1" | Remove-AzDataLakeGen2Item -Force +``` + +This command removes all items in a Filesystem with pipeline. + +## PARAMETERS + +### -AsJob +Run cmdlet in the background + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Force +Force to remove the Filesystem and all content in it + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +Azure Datalake Gen2 Item Object to remove. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item +Parameter Sets: ItemPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -PassThru +Return whether the specified Filesystem is successfully removed + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be removed. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/' + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### System.Boolean + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Set-AzDataLakeGen2ItemACLObject.md b/src/Storage/Storage.Management/help/Set-AzDataLakeGen2ItemACLObject.md new file mode 100644 index 0000000000..a3cbb0c6c8 --- /dev/null +++ b/src/Storage/Storage.Management/help/Set-AzDataLakeGen2ItemACLObject.md @@ -0,0 +1,165 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/set-azdatalakegen2itemaclobject +schema: 2.0.0 +--- + +# Set-AzDataLakeGen2ItemAclObject + +## SYNOPSIS +Creates/Updates a DataLake gen2 item ACL object, which can be used in Update-AzDataLakeGen2Item cmdlet. + +## SYNTAX + +``` +Set-AzDataLakeGen2ItemAclObject [-EntityId ] [-DefaultScope] -Permission + [-InputObject ] -AccessControlType [] +``` + +## DESCRIPTION +The **Set-AzDataLakeGen2ItemAclObject** cmdlet creates/updates a DataLake gen2 item ACL object, which can be used in Update-AzDataLakeGen2Item cmdlet. +If the new ACL entry with same AccessControlType/EntityId/DefaultScope not exist in the input ACL, will create a new ACL entry, else update permission of existing ACL entry. + +## EXAMPLES + +### Example 1: Create an ACL object with 3 ACL entry, and update ACL on a directory +``` +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rwx -DefaultScope +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "rw-" -InputObject $acl +PS C:\>Update-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/dir3" -ACL $acl + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/dir3 True 2020-03-23 09:34:31Z rwxrw-rw-+ $superuser $superuser +``` + +This command creates an ACL object with 3 ACL entries (use -InputObject parameter to add acl entry to existing acl object), and updates ACL on a directory. + +### Example 2: Create an ACL object with 4 ACL entries, and update permission of an existing ACL entry +``` +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rwx -DefaultScope +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "rw-" -InputObject $acl +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType user -EntityId $id -Permission rwx -InputObject $acl +PS C:\>$acl + +DefaultScope AccessControlType EntityId Permissions +------------ ----------------- -------- ----------- +True User rwx +False Group rw- +False Other rw- +False User ********-****-****-****-************ rwx + +PS C:\>$acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType user -EntityId $id -Permission r-x -InputObject $acl +PS C:\>$acl + +DefaultScope AccessControlType EntityId Permissions +------------ ----------------- -------- ----------- +True User rwx +False Group rw- +False Other rw- +False User ********-****-****-****-************ r-x +``` + +This command first creates an ACL object with 4 ACL entries, then run the cmdlet again with different permission but same AccessControlType/EntityId/DefaultScope of an existing ACL entry. +Then the permission of the ACL entry is updated, but no new ACL entry is added. + +## PARAMETERS + +### -AccessControlType +There are four types: "user" grants rights to the owner or a named user, "group" grants rights to the owning group or a named group, "mask" restricts rights granted to named users and the members of groups, and "other" grants rights to all users not found in any of the other entries. + +```yaml +Type: Azure.Storage.Files.DataLake.Models.AccessControlType +Parameter Sets: (All) +Aliases: +Accepted values: User, Group, Mask, Other + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DefaultScope +Set this parameter to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -EntityId +The user or group identifier. +It is omitted for entries of AccessControlType "mask" and "other". +The user or group identifier is also omitted for the owner and owning group. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +If input the PSPathAccessControlEntry\[\] object, will add the new ACL as a new element of the input PSPathAccessControlEntry\[\] object. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSPathAccessControlEntry[] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Permission +The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. +If access is not granted, the '-' character is used to denote that the permission is denied. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### None + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSPathAccessControlEntry + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Management/help/Update-AzDataLakeGen2Item.md b/src/Storage/Storage.Management/help/Update-AzDataLakeGen2Item.md new file mode 100644 index 0000000000..fd9eda2f3e --- /dev/null +++ b/src/Storage/Storage.Management/help/Update-AzDataLakeGen2Item.md @@ -0,0 +1,374 @@ +--- +external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml +Module Name: Az.Storage +online version: https://docs.microsoft.com/en-us/powershell/module/az.storage/update-azdatalakegen2item +schema: 2.0.0 +--- + +# Update-AzDataLakeGen2Item + +## SYNOPSIS +Update a file or directory on properties, metadata, permission, ACL, and owner. + +## SYNTAX + +### ReceiveManual (Default) +``` +Update-AzDataLakeGen2Item [-FileSystem] [-Path ] [-Permission ] [-Owner ] + [-Group ] [-Property ] [-Metadata ] [-Acl ] + [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +### ItemPipeline +``` +Update-AzDataLakeGen2Item -InputObject [-Permission ] [-Owner ] + [-Group ] [-Property ] [-Metadata ] [-Acl ] + [-Context ] [-DefaultProfile ] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +The **Update-AzDataLakeGen2Item** cmdlet updates a file or directory on properties, metadata, permission, ACL, and owner. +This cmdlet only works if Hierarchical Namespace is enabled for the Storage account. This kind of account can be created by run "New-AzStorageAccount" cmdlet with "-EnableHierarchicalNamespace $true". + +## EXAMPLES + +### Example 1: Create an ACL object with 3 ACL entry, and update ACL to all items in a Filesystem recursively +``` +PS C:\>$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rwx +PS C:\>$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl +PS C:\>$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "rw-" -InputObject $acl +PS C:\>Get-AzDataLakeGen2ChildItem -FileSystem "filesystem1" -Recurse | Update-AzDataLakeGen2Item -ACL $acl + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1 True 2020-03-13 13:07:34Z rwxrw-rw- $superuser $superuser +dir1/file1 False 1024 2020-03-23 09:29:18Z rwxrw-rw- $superuser $superuser +dir2 True 2020-03-23 09:28:36Z rwxrw-rw- $superuser $superuser +``` + +This command first creates an ACL object with 3 acl entry (use -InputObject parameter to add acl entry to existing acl object), then get all items in a filesystem and update acl on the items. + +### Example 2: Update all properties on a file, and show them +``` +PS C:\> $file = Update-AzDataLakeGen2Item -FileSystem "filesystem1" -Path "dir1/file1" ` + -Acl $acl ` + -Property @{"ContentType" = "image/jpeg"; "ContentMD5" = "i727sP7HigloQDsqadNLHw=="; "ContentEncoding" = "UDF8"; "CacheControl" = "READ"; "ContentDisposition" = "True"; "ContentLanguage" = "EN-US"} ` + -Metadata @{"tag1" = "value1"; "tag2" = "value2" } ` + -Permission rw-rw-rwx ` + -Owner '$superuser' ` + -Group '$superuser' + +PS C:\> $file + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/file1 False 1024 2020-03-23 09:57:33Z rwxrw-rw- $superuser $superuser + +PS C:\> $file.ACL + +DefaultScope AccessControlType EntityId Permissions +------------ ----------------- -------- ----------- +False User rwx +False Group rw- +False Other rw- + +PS C:\> $file.Permissions + +Owner : Execute, Write, Read +Group : Write, Read +Other : Write, Read +StickyBit : False +ExtendedAcls : False + +PS C:\> $file.Properties.Metadata + +Key Value +--- ----- +tag2 value2 +tag1 value1 + +PS C:\> $file.Properties + + +LastModified : 3/23/2020 9:57:33 AM +00:00 +CreatedOn : 3/23/2020 9:29:18 AM +00:00 +Metadata : {[tag2, value2], [tag1, value1]} +CopyCompletedOn : 1/1/0001 12:00:00 AM +00:00 +CopyStatusDescription : +CopyId : +CopyProgress : +CopySource : +CopyStatus : Pending +IsIncrementalCopy : False +LeaseDuration : Infinite +LeaseState : Available +LeaseStatus : Unlocked +ContentLength : 1024 +ContentType : image/jpeg +ETag : "0x8D7CF109B9878CC" +ContentHash : {139, 189, 187, 176...} +ContentEncoding : UDF8 +ContentDisposition : True +ContentLanguage : EN-US +CacheControl : READ +AcceptRanges : bytes +IsServerEncrypted : True +EncryptionKeySha256 : +AccessTier : Cool +ArchiveStatus : +AccessTierChangedOn : 1/1/0001 12:00:00 AM +00:00 +``` + +This command updates all properties on a file (ACL, permission,owner, group, metadata, property can be updated with any conbination), and show them in Powershell console. + +### Example 3: Add an ACL entry to a directory +``` +## Get the origin ACL +PS C:\> $acl = (Get-AzDataLakeGen2Item -FileSystem "filesystem1" -Path 'dir1/dir3/').ACL + +# Update permission of a new ACL entry (if ACL entry with same AccessControlType/EntityId/DefaultScope not exist, will add a new ACL entry, else update permission of existing ACL entry) +PS C:\> $acl = Set-AzDataLakeGen2ItemAclObject -AccessControlType user -EntityId $id -Permission rw- -InputObject $acl + +# set the new acl to the directory +PS C:\> update-AzDataLakeGen2Item -FileSystem "filesystem1" -Path 'dir1/dir3/' -ACL $acl + + FileSystem Name: filesystem1 + +Path IsDirectory Length LastModified Permissions Owner Group +---- ----------- ------ ------------ ----------- ----- ----- +dir1/dir3 True 2020-03-23 09:34:31Z rwxrw-rw-+ $superuser $superuser +``` + +This command gets ACL from a directory, updates/adds an ACL entry, and sets back to the directory. +If ACL entry with same AccessControlType/EntityId/DefaultScope not exist, will add a new ACL entry, else update permission of existing ACL entry. + +## PARAMETERS + +### -Acl +Sets POSIX access control rights on files and directories. +Create this object with New-AzDataLakeGen2ItemAclObject. + +```yaml +Type: Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSPathAccessControlEntry[] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Context +Azure Storage Context Object + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByPropertyName, ByValue) +Accept wildcard characters: False +``` + +### -DefaultProfile +The credentials, account, tenant, and subscription used for communication with Azure. + +```yaml +Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer +Parameter Sets: (All) +Aliases: AzureRmContext, AzureCredential + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -FileSystem +FileSystem name + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: True +Position: 0 +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Group +Sets the owning group of the blob. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +Azure Datalake Gen2 Item Object to update + +```yaml +Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item +Parameter Sets: ItemPipeline +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Metadata +Specifies metadata for the directory or file. + +```yaml +Type: System.Collections.Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Owner +Sets the owner of the blob. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Path +The path in the specified Filesystem that should be updated. +Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/'. +Not specify this parameter will update the root directory of the Filesystem. + +```yaml +Type: System.String +Parameter Sets: ReceiveManual +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -Permission +Sets POSIX access permissions for the file owner, the file owning group, and others. +Each class may be granted read, write, or execute permission. +Symbolic (rwxrw-rw-) is supported. +Invalid in conjunction with Acl. + +```yaml +Type: System.String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Property +Specifies properties for the directory or file. +The supported properties for file are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentMD5, ContentType. +The supported properties for directory are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage. + +```yaml +Type: System.Collections.Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: System.Management.Automation.SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.String + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext + +## OUTPUTS + +### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + +## NOTES + +## RELATED LINKS diff --git a/src/Storage/Storage.Test/Storage.Test.csproj b/src/Storage/Storage.Test/Storage.Test.csproj index 5bba25da9f..6d346883d1 100644 --- a/src/Storage/Storage.Test/Storage.Test.csproj +++ b/src/Storage/Storage.Test/Storage.Test.csproj @@ -14,9 +14,9 @@ - - - + + + diff --git a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlob.cs b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlob.cs index ce33911cf4..852493d185 100644 --- a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlob.cs +++ b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlob.cs @@ -134,30 +134,6 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet Channel = channel; } - /// - /// get the CloudBlobContainer object by name if container exists - /// - /// container name - /// return CloudBlobContianer object if specified container exists, otherwise throw an exception - internal async Task GetCloudBlobContainerByName(IStorageBlobManagement localChannel, string containerName, bool skipCheckExists = false) - { - if (!NameUtil.IsValidContainerName(containerName)) - { - throw new ArgumentException(String.Format(Resources.InvalidContainerName, containerName)); - } - - BlobRequestOptions requestOptions = RequestOptions; - CloudBlobContainer container = localChannel.GetContainerReference(containerName); - - if (!skipCheckExists && container.ServiceClient.Credentials.IsSharedKey - && !await localChannel.DoesContainerExistAsync(container, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false)) - { - throw new ArgumentException(String.Format(Resources.ContainerNotFound, containerName)); - } - - return container; - } - /// /// list blobs by blob name and container name /// diff --git a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageContainer.cs b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageContainer.cs index d2013853de..1e2011e38c 100644 --- a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageContainer.cs +++ b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageContainer.cs @@ -29,7 +29,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet /// List azure storage container /// [Cmdlet("Get", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageContainer", DefaultParameterSetName = NameParameterSet),OutputType(typeof(AzureStorageContainer))] - [Alias("Get-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageContainerAcl")] + [Alias("Get-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageContainerAcl", "Get-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DatalakeGen2FileSystem")] public class GetAzureStorageContainerCommand : StorageCloudBlobCmdletBase { /// diff --git a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainer.cs b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainer.cs index a4ce0cb156..9a37728a09 100644 --- a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainer.cs +++ b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainer.cs @@ -27,6 +27,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet /// create a new azure container /// [Cmdlet("New", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageContainer"),OutputType(typeof(AzureStorageContainer))] + [Alias("New-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DatalakeGen2FileSystem")] public class NewAzureStorageContainerCommand : StorageCloudBlobCmdletBase { [Alias("N", "Container")] diff --git a/src/Storage/Storage/Blob/Cmdlet/RemoveAzureStorageContainer.cs b/src/Storage/Storage/Blob/Cmdlet/RemoveAzureStorageContainer.cs index 6767a7be7e..96e8f86979 100644 --- a/src/Storage/Storage/Blob/Cmdlet/RemoveAzureStorageContainer.cs +++ b/src/Storage/Storage/Blob/Cmdlet/RemoveAzureStorageContainer.cs @@ -27,6 +27,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet /// remove specified azure container /// [Cmdlet("Remove", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageContainer", SupportsShouldProcess = true),OutputType(typeof(Boolean))] + [Alias("Remove-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DatalakeGen2FileSystem")] public class RemoveAzureStorageContainerCommand : StorageCloudBlobCmdletBase { [Alias("N", "Container")] diff --git a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs index 5de329e6ff..7a192291af 100644 --- a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs +++ b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs @@ -367,93 +367,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob } base.DoEndProcessing(); - } - - //only support the common blob properties for block blob and page blob - //http://msdn.microsoft.com/en-us/library/windowsazure/ee691966.aspx - private Dictionary> validCloudBlobProperties = - new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - {"CacheControl", (p, v) => p.CacheControl = v}, - {"ContentDisposition", (p, v) => p.ContentDisposition = v}, - {"ContentEncoding", (p, v) => p.ContentEncoding = v}, - {"ContentLanguage", (p, v) => p.ContentLanguage = v}, - {"ContentMD5", (p, v) => p.ContentMD5 = v}, - {"ContentType", (p, v) => p.ContentType = v}, - }; - - /// - /// check whether the blob properties is valid - /// - /// Blob properties table - private void ValidateBlobProperties(Hashtable properties) - { - if (properties == null) - { - return; - } - - foreach (DictionaryEntry entry in properties) - { - if (!validCloudBlobProperties.ContainsKey(entry.Key.ToString())) - { - throw new ArgumentException(String.Format(Resources.InvalidBlobProperties, entry.Key.ToString(), entry.Value.ToString())); - } - } - } - - /// - /// set blob properties to a blob object - /// - /// CloudBlob object - /// blob properties hashtable - private void SetBlobProperties(StorageBlob.CloudBlob blob, Hashtable properties) - { - if (properties == null) - { - return; - } - - foreach (DictionaryEntry entry in properties) - { - string key = entry.Key.ToString(); - string value = entry.Value.ToString(); - Action action = validCloudBlobProperties[key]; - - if (action != null) - { - action(blob.Properties, value); - } - } - } - - /// - /// set blob metadata to a blob object - /// - /// CloudBlob object - /// meta data hashtable - private void SetBlobMeta(StorageBlob.CloudBlob blob, Hashtable meta) - { - if (meta == null) - { - return; - } - - foreach (DictionaryEntry entry in meta) - { - string key = entry.Key.ToString(); - string value = entry.Value.ToString(); - - if (blob.Metadata.ContainsKey(key)) - { - blob.Metadata[key] = value; - } - else - { - blob.Metadata.Add(key, value); - } - } - } + } /// /// set blob AccessTier diff --git a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs index 9b8a0b3038..08090bf022 100644 --- a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs +++ b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs @@ -23,6 +23,14 @@ namespace Microsoft.WindowsAzure.Commands.Storage using System; using System.Collections.Generic; using System.Globalization; + using System.Threading.Tasks; + using System.Collections; + using Microsoft.Azure.Commands.Common.Authentication.Abstractions; + using global::Azure.Storage.Blobs; + using global::Azure.Storage.Files.DataLake; + using global::Azure.Storage; + using global::Azure; + using global::Azure.Storage.Files.DataLake.Models; /// /// Base cmdlet for storage blob/container cmdlet @@ -209,12 +217,49 @@ namespace Microsoft.WindowsAzure.Commands.Storage /// IStorageBlobManagement channel object internal void WriteCloudBlobObject(long taskId, IStorageBlobManagement channel, CloudBlob blob, BlobContinuationToken continuationToken = null) { - AzureStorageBlob azureBlob = new AzureStorageBlob(blob); - azureBlob.Context = channel.StorageContext; + AzureStorageBlob azureBlob = new AzureStorageBlob(blob, channel.StorageContext); azureBlob.ContinuationToken = continuationToken; OutputStream.WriteObject(taskId, azureBlob); } + /// + /// Write a datalake gen2 file to output + /// + internal void WriteDataLakeGen2Item(IStorageBlobManagement channel, DataLakeFileClient fileClient, long? taskId = null) + { + AzureDataLakeGen2Item azureDataLakeGen2Item = new AzureDataLakeGen2Item(fileClient); + azureDataLakeGen2Item.Context = channel.StorageContext; + if (taskId == null) + { + WriteObject(azureDataLakeGen2Item); + } + else + { + OutputStream.WriteObject(taskId.Value, azureDataLakeGen2Item); + } + } + + /// + /// Write a datalake gen2 folder to output. + /// + internal void WriteDataLakeGen2Item(IStorageBlobManagement channel, DataLakeDirectoryClient dirClient) + { + AzureDataLakeGen2Item azureDataLakeGen2Item = new AzureDataLakeGen2Item(dirClient); + azureDataLakeGen2Item.Context = channel.StorageContext; + WriteObject(azureDataLakeGen2Item); + } + + /// + /// Write a datalake gen2 pathitem to output. + /// + internal void WriteDataLakeGen2Item(IStorageBlobManagement channel, PathItem item, DataLakeFileSystemClient fileSystem, string ContinuationToken = null, bool fetchProperties = false) + { + AzureDataLakeGen2Item azureDataLakeGen2Item = new AzureDataLakeGen2Item(item, fileSystem, fetchProperties); + azureDataLakeGen2Item.Context = channel.StorageContext; + azureDataLakeGen2Item.ContinuationToken = ContinuationToken; + WriteObject(azureDataLakeGen2Item); + } + /// /// Write CloudBlob to output using specified service channel /// @@ -269,11 +314,352 @@ namespace Microsoft.WindowsAzure.Commands.Storage return !(listedBlobs.MoveNext() && listedBlobs.Current != null); } } - catch(Exception) + catch (Exception) { return false; } } + /// + /// decide if a object represent a folder of datalake gen2 + /// + /// the PathProperties of the datalakeGen2 Object + /// return true if it represent a folder of datalake gen2 + public static bool isDirectory(PathProperties fileProperties) + { + if (fileProperties.Metadata.Contains(new KeyValuePair("hdi_isfolder", "true")) + && fileProperties.ContentLength == 0) + { + return true; + } + return false; + } + + /// + /// get the CloudBlobContainer object by name if container exists + /// + /// container name + /// return CloudBlobContianer object if specified container exists, otherwise throw an exception + internal async Task GetCloudBlobContainerByName(IStorageBlobManagement localChannel, string containerName, bool skipCheckExists = false) + { + if (!NameUtil.IsValidContainerName(containerName)) + { + throw new ArgumentException(String.Format(Resources.InvalidContainerName, containerName)); + } + + BlobRequestOptions requestOptions = RequestOptions; + CloudBlobContainer container = localChannel.GetContainerReference(containerName); + + if (!skipCheckExists && container.ServiceClient.Credentials.IsSharedKey + && !await localChannel.DoesContainerExistAsync(container, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false)) + { + throw new ArgumentException(String.Format(Resources.ContainerNotFound, containerName)); + } + + return container; + } + + /// + /// Get an Exist DataLakeGen2Item, return true is the item is a folder, return false if it's File + /// + /// the blob container + /// the path of the Items + /// return true if the item is a folder, else false + public static bool GetExistDataLakeGen2Item(DataLakeFileSystemClient fileSystem, string path, out DataLakeFileClient fileClient, out DataLakeDirectoryClient dirClient) + { + try + { + if (string.IsNullOrEmpty(path)) + { + dirClient = fileSystem.GetDirectoryClient(""); + fileClient = null; + return true; + } + + fileClient = fileSystem.GetFileClient(path); + PathProperties properties = fileClient.GetProperties().Value; + if (isDirectory(properties)) + { + dirClient = fileSystem.GetDirectoryClient(path); + fileClient = null; + return true; + } + else + { + dirClient = null; + return false; + } + } + catch (RequestFailedException e) when (e.Status == 404) + { + // TODO: through exception that the item not exist + throw new ArgumentException(string.Format("The Item in File System {0} on path {1} does not exist.", fileSystem.Name, path)); + } + } + + //only support the common properties for DatalakeGen2File + protected static Dictionary> validDatalakeGen2FileProperties = + new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + {"CacheControl", (p, v) => p.CacheControl = v}, + {"ContentDisposition", (p, v) => p.ContentDisposition = v}, + {"ContentEncoding", (p, v) => p.ContentEncoding = v}, + {"ContentLanguage", (p, v) => p.ContentLanguage = v}, + {"ContentMD5", (p, v) => p.ContentHash = Convert.FromBase64String(v)}, + {"ContentType", (p, v) => p.ContentType = v}, + }; + + //only support the common properties for DatalakeGen2Folder + protected static Dictionary> validDatalakeGen2FolderProperties = + new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + {"CacheControl", (p, v) => p.CacheControl = v}, + {"ContentDisposition", (p, v) => p.ContentDisposition = v}, + {"ContentEncoding", (p, v) => p.ContentEncoding = v}, + {"ContentLanguage", (p, v) => p.ContentLanguage = v}, + }; + + /// + /// Set properties to a datalake gen2 Datalakegen2Item + /// + /// datalake gen2 Datalakegen2Item + /// properties to set + /// True will set to server, false only set to the local Datalakegen2Item object + protected static PathHttpHeaders SetDatalakegen2ItemProperties(DataLakePathClient item, Hashtable BlobProperties, bool setToServer = true) + { + if (BlobProperties != null) + { + // Valid Blob Dir properties + foreach (DictionaryEntry entry in BlobProperties) + { + if (!validDatalakeGen2FileProperties.ContainsKey(entry.Key.ToString())) + { + throw new ArgumentException(String.Format("InvalidDataLakeFileProperties", entry.Key.ToString(), entry.Value.ToString())); + } + } + + PathHttpHeaders headers = new PathHttpHeaders(); + foreach (DictionaryEntry entry in BlobProperties) + { + string key = entry.Key.ToString(); + string value = entry.Value.ToString(); + Action action = validDatalakeGen2FileProperties[key]; + + if (action != null) + { + action(headers, value); + } + } + if (setToServer && item != null) + { + item.SetHttpHeaders(headers); + } + return headers; + } + else + { + return null; + } + } + + /// + /// Set Metadata to a datalake gen2 item + /// + /// datalake gen2 item + /// Metadata to set + /// True will set to server, false only set to the local Datalakegen2Item object + protected static IDictionary SetDatalakegen2ItemMetaData(DataLakePathClient item, Hashtable Metadata, bool setToServer = true) + { + if (Metadata != null) + { + IDictionary metadata = GetUpdatedMetaData(Metadata, null); + if (setToServer && item != null) + { + item.SetMetadata(metadata); + } + return metadata; + } + else + { + return null; + } + } + + public static IDictionary GetUpdatedMetaData(Hashtable Metadata, IDictionary originalMetadata = null) + { + if (Metadata != null) + { + IDictionary metadata; + if (originalMetadata == null) + { + metadata = new Dictionary(); + } + else + { + metadata = originalMetadata; + metadata.Remove("hdi_isfolder"); + } + foreach (DictionaryEntry entry in Metadata) + { + string key = entry.Key.ToString(); + string value = entry.Value.ToString(); + + if (metadata.ContainsKey(key)) + { + metadata[key] = value; + } + else + { + metadata.Add(key, value); + } + } + return metadata; + } + else + { + return originalMetadata; + } + } + + /// + /// Get Item string without SAS for confirmation string. + /// + /// + /// + protected static string GetDataLakeItemUriWithoutSas(DataLakePathClient item) + { + string uriString = item.Uri.ToString(); + int length = uriString.IndexOf("?"); + if (length < 0) // Not container "?" + { + return uriString; + } + else + { + return uriString.Substring(0, uriString.IndexOf("?")); + } + } + + /// + /// get the DataLakeFileSystemClient object by name if DataLakeFileSystem exists + /// + /// DataLakeFileSystem name + /// return DataLakeFileSystemClient object if specified DataLakeFileSystem exists, otherwise throw an exception + internal DataLakeFileSystemClient GetFileSystemClientByName(IStorageBlobManagement localChannel, string fileSystemName, bool skipCheckExists = false) + { + if (!NameUtil.IsValidContainerName(fileSystemName)) + { + throw new ArgumentException(String.Format(Resources.InvalidContainerName, fileSystemName)); + } + + Uri fileSystemUri = localChannel.StorageContext.StorageAccount.CreateCloudBlobClient().GetContainerReference(fileSystemName).Uri; + DataLakeFileSystemClient fileSystem; + + if (localChannel.StorageContext.StorageAccount.Credentials.IsToken) //Oauth + { + fileSystem = new DataLakeFileSystemClient(fileSystemUri, localChannel.StorageContext.Track2OauthToken); + } + else if (localChannel.StorageContext.StorageAccount.Credentials.IsSAS) //SAS + { + fileSystem = new DataLakeFileSystemClient(new Uri (fileSystemUri.ToString() + localChannel.StorageContext.StorageAccount.Credentials.SASToken)); + } + else if (localChannel.StorageContext.StorageAccount.Credentials.IsSharedKey) //Shared Key + { + fileSystem = new DataLakeFileSystemClient(fileSystemUri, + new StorageSharedKeyCredential(localChannel.StorageContext.StorageAccountName, localChannel.StorageContext.StorageAccount.Credentials.ExportBase64EncodedKey())); + } + else //Anonymous + { + fileSystem = new DataLakeFileSystemClient(fileSystemUri); + } + + return fileSystem; + } + + //only support the common blob properties for block blob and page blob + //http://msdn.microsoft.com/en-us/library/windowsazure/ee691966.aspx + protected static Dictionary> validCloudBlobProperties = + new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + {"CacheControl", (p, v) => p.CacheControl = v}, + {"ContentDisposition", (p, v) => p.ContentDisposition = v}, + {"ContentEncoding", (p, v) => p.ContentEncoding = v}, + {"ContentLanguage", (p, v) => p.ContentLanguage = v}, + {"ContentMD5", (p, v) => p.ContentMD5 = v}, + {"ContentType", (p, v) => p.ContentType = v}, + }; + + /// + /// check whether the blob properties is valid + /// + /// Blob properties table + protected void ValidateBlobProperties(Hashtable properties) + { + if (properties == null) + { + return; + } + + foreach (DictionaryEntry entry in properties) + { + if (!validCloudBlobProperties.ContainsKey(entry.Key.ToString())) + { + throw new ArgumentException(String.Format(Resources.InvalidBlobProperties, entry.Key.ToString(), entry.Value.ToString())); + } + } + } + + /// + /// set blob properties to a blob object + /// + /// CloudBlob object + /// blob properties hashtable + protected static void SetBlobProperties(CloudBlob blob, Hashtable properties) + { + if (properties == null) + { + return; + } + + foreach (DictionaryEntry entry in properties) + { + string key = entry.Key.ToString(); + string value = entry.Value.ToString(); + Action action = validCloudBlobProperties[key]; + + if (action != null) + { + action(blob.Properties, value); + } + } + } + + /// + /// set blob metadata to a blob object + /// + /// CloudBlob object + /// meta data hashtable + protected static void SetBlobMeta(CloudBlob blob, Hashtable meta) + { + if (meta == null) + { + return; + } + + foreach (DictionaryEntry entry in meta) + { + string key = entry.Key.ToString(); + string value = entry.Value.ToString(); + + if (blob.Metadata.ContainsKey(key)) + { + blob.Metadata[key] = value; + } + else + { + blob.Metadata.Add(key, value); + } + } + } } } \ No newline at end of file diff --git a/src/Storage/Storage/Common/AzureDataLakeGen2Item.cs b/src/Storage/Storage/Common/AzureDataLakeGen2Item.cs new file mode 100644 index 0000000000..faf827946b --- /dev/null +++ b/src/Storage/Storage/Common/AzureDataLakeGen2Item.cs @@ -0,0 +1,188 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel +{ + using System; + using Microsoft.WindowsAzure.Commands.Common.Attributes; + using Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel; + using global::Azure.Storage.Files.DataLake; + using global::Azure.Storage.Files.DataLake.Models; + + /// + /// Azure storage blob object + /// + public class AzureDataLakeGen2Item : AzureStorageBase + { + /// + /// File Properties + /// + public DataLakeFileClient File { get; set; } + + /// + /// Directory Properties + /// + public DataLakeDirectoryClient Directory { get; private set; } + + /// + /// The Path of the item + /// + [Ps1Xml(Label = "Path", Target = ViewControl.Table, Position = 0, TableColumnWidth = 20)] + public string Path { get; set; } + + /// + /// CloudBlobDirectory object + /// + [Ps1Xml(Label = "IsDirectory", Target = ViewControl.Table, Position = 1, TableColumnWidth = 12)] + public bool IsDirectory { get; private set; } + + /// + /// Datalake Gen2 Item path Permissions + /// + [Ps1Xml(Label = "Permissions", Target = ViewControl.Table, ScriptBlock = "$_.Permissions.ToSymbolicPermissions()", Position = 5, TableColumnWidth = 12)] + public PathPermissions Permissions { get; set; } + + /// + /// Datalake Gen2 Item ACL + /// + public PSPathAccessControlEntry[] ACL { get; set; } + + /// + /// Datalake Item PathProperties + /// + public PathProperties Properties { get; private set; } + + /// + /// Datalake Item PathAccessControl + /// + public PathAccessControl AccessControl { get; private set; } + + /// + /// Datalake Item list ContinuationToken + /// + public string ContinuationToken { get; set; } + + /// + /// Blob length + /// + [Ps1Xml(Label = "Length", Target = ViewControl.Table, ScriptBlock = "if ($_.IsDirectory -eq $false) {$_.Length}", Position = 2, TableColumnWidth = 15)] + public long Length { get; private set; } + + /// + /// Blob content type + /// + [Ps1Xml(Label = "ContentType", Target = ViewControl.Table, Position = 3, TableColumnWidth = 30)] + public string ContentType { get; private set; } + + /// + /// Blob last modified time + /// + [Ps1Xml(Label = "LastModified", Target = ViewControl.Table, ScriptBlock = "$_.LastModified.UtcDateTime.ToString(\"u\")", Position = 4, TableColumnWidth = 20)] + public DateTimeOffset? LastModified { get; private set; } + + /// + /// The owner of the item + /// + [Ps1Xml(Label = "Owner", Target = ViewControl.Table, Position = 6, TableColumnWidth = 10)] + public string Owner { get; set; } + + /// + /// The Group of the item + /// + [Ps1Xml(Label = "Group", Target = ViewControl.Table, Position = 7, TableColumnWidth = 10)] + public string Group { get; set; } + + /// + /// Azure DataLakeGen2 Item constructor + /// + /// CloudBlockBlob blob object + public AzureDataLakeGen2Item(DataLakeFileClient fileClient) + { + Name = fileClient.Name; + Path = fileClient.Path; + File = fileClient; + Properties = fileClient.GetProperties(); + AccessControl = File.GetAccessControl(); + Length = Properties.ContentLength; + ContentType = Properties.ContentType; + LastModified = Properties.LastModified; + IsDirectory = false; + Permissions = AccessControl.Permissions; + ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList); + Owner = AccessControl.Owner; + Group = AccessControl.Group; + } + + /// + /// Azure DataLakeGen2 Item constructor + /// + /// Cloud blob Directory object + public AzureDataLakeGen2Item(DataLakeDirectoryClient directoryClient) + { + Name = directoryClient.Name; + Path = directoryClient.Path; + Directory = directoryClient; + IsDirectory = true; + if (directoryClient.Path != "/") //if root directory, GetProperties() will fail. Skip until this is fixed. + { + Properties = directoryClient.GetProperties(); + Length = Properties.ContentLength; + ContentType = Properties.ContentType; + LastModified = Properties.LastModified; + } + AccessControl = directoryClient.GetAccessControl(); + Permissions = AccessControl.Permissions; + ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList); + Owner = AccessControl.Owner; + Group = AccessControl.Group; + } + + + /// + /// Azure DataLakeGen2 Item constructor + /// + /// datalake gen2 listout item + public AzureDataLakeGen2Item(PathItem item, DataLakeFileSystemClient fileSystem, bool fetchProperties = false) + { + this.Name = item.Name; + this.Path = item.Name; + this.IsDirectory = item.IsDirectory is null ? false : item.IsDirectory.Value; + DataLakePathClient pathclient = null; + if (this.IsDirectory) // Directory + { + this.Directory = fileSystem.GetDirectoryClient(item.Name); + pathclient = this.Directory; + } + else //File + { + this.File = fileSystem.GetFileClient(item.Name); + pathclient = this.File; + } + + this.Owner = item.Owner; + this.Group = item.Group; + this.Permissions = PathPermissions.ParseSymbolicPermissions(item.Permissions); + this.LastModified = item.LastModified; + this.Length = item.ContentLength is null ? 0 : item.ContentLength.Value; + + if (fetchProperties) + { + this.Properties = pathclient.GetProperties(); + this.AccessControl = pathclient.GetAccessControl(); + this.ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(this.AccessControl.AccessControlList); + this.ContentType = Properties.ContentType; + } + } + } +} diff --git a/src/Storage/Storage/Common/AzureSessionCredential.cs b/src/Storage/Storage/Common/AzureSessionCredential.cs new file mode 100644 index 0000000000..ec6625a51a --- /dev/null +++ b/src/Storage/Storage/Common/AzureSessionCredential.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Microsoft.Azure.Commands.Common.Authentication; +using Microsoft.Azure.Commands.Common.Authentication.Abstractions; + +namespace Microsoft.WindowsAzure.Commands.Storage.Common +{ + public delegate void DebugLogWriter(string log); + public class AzureSessionCredential : TokenCredential + { + public AzureSessionCredential(IAzureContext DefaultContext, DebugLogWriter logWriter = null) + { + if (DefaultContext == null || DefaultContext.Account == null) + { + throw new InvalidOperationException(Resources.ContextCannotBeNull); + } + if (logWriter != null) + { + this.debugLogWriter = logWriter; + } + + IAccessToken accessToken1 = AzureSession.Instance.AuthenticationFactory.Authenticate( + DefaultContext.Account, + EnsureStorageOAuthAudienceSet(DefaultContext.Environment), + DefaultContext.Tenant.Id, + null, + ShowDialog.Never, + null, + StorageOAuthEndpointResourceKey); + accessToken = accessToken1; + } + + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + AccessToken token; + accessToken.AuthorizeRequest((tokenType, tokenValue) => + { + token = new AccessToken(tokenValue, DateTimeOffset.UtcNow); + }); + if (this.debugLogWriter != null) + { + this.debugLogWriter("[" + DateTime.Now.ToString() + "] GetToken: " + token.Token); + } + return token; + } + + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + AccessToken token; + accessToken.AuthorizeRequest((tokenType, tokenValue) => + { + token = new AccessToken(tokenValue, DateTimeOffset.UtcNow); + }); + + if (this.debugLogWriter != null) + { + this.debugLogWriter("[" + DateTime.Now.ToString() + "] GetTokenAsync: " + token.Token); + } + return new ValueTask(token); + } + + private IAzureEnvironment EnsureStorageOAuthAudienceSet(IAzureEnvironment environment) + { + if (environment != null) + { + if (!environment.IsPropertySet(StorageOAuthEndpointResourceKey)) + { + environment.SetProperty(StorageOAuthEndpointResourceKey, StorageOAuthEndpointResourceValue); + } + } + + return environment; + } + + /// + /// The extension key to use for the storage token audience value + /// + public const string StorageOAuthEndpointResourceKey = "StorageOAuthEndpointResourceId"; + + /// + /// Default resourceId for storage OAuth tokens + /// + public const string StorageOAuthEndpointResourceValue = "https://storage.azure.com"; + + + private IAccessToken accessToken; + private DebugLogWriter debugLogWriter = null; + } + +} diff --git a/src/Storage/Storage/Common/AzureStorageBlob.cs b/src/Storage/Storage/Common/AzureStorageBlob.cs index 6785ab985d..350d612f72 100644 --- a/src/Storage/Storage/Common/AzureStorageBlob.cs +++ b/src/Storage/Storage/Common/AzureStorageBlob.cs @@ -17,6 +17,11 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel using Microsoft.Azure.Storage.Blob; using System; using Microsoft.WindowsAzure.Commands.Common.Attributes; + using global::Azure.Storage.Blobs; + using Microsoft.WindowsAzure.Commands.Storage; + using Microsoft.Azure.Commands.Common.Authentication.Abstractions; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using global::Azure.Storage; /// /// Azure storage blob object @@ -49,6 +54,38 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel [Ps1Xml(Label = "IsDeleted", Target = ViewControl.Table, Position = 7, TableColumnWidth = 10)] public bool IsDeleted { get; private set; } + /// + /// XSCL Track2 Blob Client, used to run blob APIs + /// + public BlobClient BlobClient + { + get + { + if (privateBlobClient == null) + { + privateBlobClient = GetTrack2BlobClient(this.ICloudBlob, (AzureStorageContext)this.Context); + } + return privateBlobClient; + } + } + private BlobClient privateBlobClient = null; + + /// + /// XSCL Track2 Blob properties, will retrieve the properties on server and return to user + /// + public global::Azure.Storage.Blobs.Models.BlobProperties BlobProperties + { + get + { + if (privateBlobProperties == null) + { + privateBlobProperties = BlobClient.GetProperties().Value; + } + return privateBlobProperties; + } + } + private global::Azure.Storage.Blobs.Models.BlobProperties privateBlobProperties = null; + /// /// Blob IsDeleted /// @@ -60,6 +97,7 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel [Ps1Xml(Label = "ContentType", Target = ViewControl.Table, Position = 3, TableColumnWidth = 30)] public string ContentType { get; private set; } + /// /// Blob last modified time /// @@ -81,7 +119,7 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel /// Azure storage blob constructor /// /// ICloud blob object - public AzureStorageBlob(CloudBlob blob) + public AzureStorageBlob(CloudBlob blob, AzureStorageContext storageContext) { Name = blob.Name; ICloudBlob = blob; @@ -92,6 +130,54 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel ContentType = blob.Properties.ContentType; LastModified = blob.Properties.LastModified; SnapshotTime = blob.SnapshotTime; + this.Context = storageContext; + } + + //refresh XSCL track2 blob properties object from server + public void FetchAttributes() + { + privateBlobProperties = BlobClient.GetProperties().Value; + } + + // Convert Track1 Blob object to Track 2 blob Client + protected static BlobClient GetTrack2BlobClient(CloudBlob cloubBlob, AzureStorageContext context) + { + BlobClient blobClient; + if (cloubBlob.ServiceClient.Credentials.IsToken) //Oauth + { + if (context == null) + { + //TODO : Get Oauth context from current login user. + throw new System.Exception("Need Storage Context to convert Track1 Blob object in token credentail to Track2 Blob object."); + } + blobClient = new BlobClient(cloubBlob.SnapshotQualifiedUri, context.Track2OauthToken); + + } + else if (cloubBlob.ServiceClient.Credentials.IsSAS) //SAS + { + string fullUri = cloubBlob.SnapshotQualifiedUri.ToString(); + if (cloubBlob.IsSnapshot) + { + // Since snapshot URL already has '?', need remove '?' in the first char of sas + fullUri = fullUri + "&" + cloubBlob.ServiceClient.Credentials.SASToken.Substring(1); + } + else + { + fullUri = fullUri + cloubBlob.ServiceClient.Credentials.SASToken; + } + blobClient = new BlobClient(new Uri(fullUri)); + } + else if (cloubBlob.ServiceClient.Credentials.IsSharedKey) //Shared Key + { + blobClient = new BlobClient(cloubBlob.SnapshotQualifiedUri, + new StorageSharedKeyCredential(context.StorageAccountName, cloubBlob.ServiceClient.Credentials.ExportBase64EncodedKey())); + } + else //Anonymous + { + blobClient = new BlobClient(cloubBlob.SnapshotQualifiedUri); + } + + return blobClient; } } } diff --git a/src/Storage/Storage/Common/AzureStorageContainer.cs b/src/Storage/Storage/Common/AzureStorageContainer.cs index 25f7f299b0..ea0e782abb 100644 --- a/src/Storage/Storage/Common/AzureStorageContainer.cs +++ b/src/Storage/Storage/Common/AzureStorageContainer.cs @@ -17,6 +17,9 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel using Microsoft.Azure.Storage.Blob; using System; using Microsoft.WindowsAzure.Commands.Common.Attributes; + using global::Azure.Storage.Blobs; + using Microsoft.WindowsAzure.Commands.Storage; + using global::Azure.Storage; /// /// azure storage container @@ -52,6 +55,38 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel /// public BlobContinuationToken ContinuationToken { get; set; } + /// + /// XSCL Track2 container Client, used to run blob APIs + /// + public BlobContainerClient BlobContainerClient + { + get + { + if (privateBlobContainerClient == null) + { + privateBlobContainerClient = GetTrack2BlobContainerClient(this.CloudBlobContainer, (AzureStorageContext)this.Context); + } + return privateBlobContainerClient; + } + } + private BlobContainerClient privateBlobContainerClient = null; + + /// + /// XSCL Track2 Blob properties, will retrieve the properties on server and return to user + /// + public global::Azure.Storage.Blobs.Models.BlobContainerProperties BlobContainerProperties + { + get + { + if (privateBlobContainerProperties == null) + { + privateBlobContainerProperties = BlobContainerClient.GetProperties().Value; + } + return privateBlobContainerProperties; + } + } + private global::Azure.Storage.Blobs.Models.BlobContainerProperties privateBlobContainerProperties = null; + /// /// init azure storage container using CloudBlobContainer and BlobContainerPermissions /// @@ -74,5 +109,44 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel LastModified = container.Properties.LastModified; } + + //refresh XSCL track2 container properties object from server + public void FetchAttributes() + { + privateBlobContainerProperties = BlobContainerClient.GetProperties().Value; + } + + // Convert Track1 Container object to Track 2 Container Client + protected static BlobContainerClient GetTrack2BlobContainerClient(CloudBlobContainer cloubContainer, AzureStorageContext context) + { + BlobContainerClient blobContainerClient; + if (cloubContainer.ServiceClient.Credentials.IsToken) //Oauth + { + if (context == null) + { + //TODO : Get Oauth context from current login user. + throw new System.Exception("Need Storage Context to convert Track1 object in token credentail to Track2 object."); + } + blobContainerClient = new BlobContainerClient(cloubContainer.Uri, context.Track2OauthToken); + + } + else if (cloubContainer.ServiceClient.Credentials.IsSAS) //SAS + { + string fullUri = cloubContainer.Uri.ToString(); + fullUri = fullUri + cloubContainer.ServiceClient.Credentials.SASToken; + blobContainerClient = new BlobContainerClient(new Uri(fullUri)); + } + else if (cloubContainer.ServiceClient.Credentials.IsSharedKey) //Shared Key + { + blobContainerClient = new BlobContainerClient(cloubContainer.Uri, + new StorageSharedKeyCredential(context.StorageAccountName, cloubContainer.ServiceClient.Credentials.ExportBase64EncodedKey())); + } + else //Anonymous + { + blobContainerClient = new BlobContainerClient(cloubContainer.Uri); + } + + return blobContainerClient; + } } } diff --git a/src/Storage/Storage/Common/AzureStorageContext.cs b/src/Storage/Storage/Common/AzureStorageContext.cs index 7422247a98..ec4da22d6a 100644 --- a/src/Storage/Storage/Common/AzureStorageContext.cs +++ b/src/Storage/Storage/Common/AzureStorageContext.cs @@ -20,6 +20,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage using XTable = Microsoft.Azure.Cosmos.Table; using System; using System.Collections.Generic; + using Microsoft.WindowsAzure.Commands.Storage.Common; /// /// Storage context @@ -73,6 +74,11 @@ namespace Microsoft.WindowsAzure.Commands.Storage /// public virtual XTable.CloudStorageAccount TableStorageAccount { get; protected set; } + /// + /// Storage Library Track2 Oauth credential + /// + public virtual AzureSessionCredential Track2OauthToken { get; set; } + /// /// Endpoint suffix (everything after "table.", "blob." or "queue.") /// @@ -136,7 +142,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage /// Create a storage context usign cloud storage account /// /// cloud storage account - public AzureStorageContext(CloudStorageAccount account, string accountName = null) + public AzureStorageContext(CloudStorageAccount account, string accountName = null, IAzureContext DefaultContext = null, DebugLogWriter logWriter = null) { StorageAccount = account; TableStorageAccount = XTable.CloudStorageAccount.Parse(StorageAccount.ToString(true)); @@ -180,6 +186,10 @@ namespace Microsoft.WindowsAzure.Commands.Storage StorageAccountName = "[Anonymous]"; } } + if (account.Credentials.IsToken) + { + Track2OauthToken = new AzureSessionCredential(DefaultContext, logWriter); + } } /// diff --git a/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs b/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs index 738c6c68b1..09d01a0b3c 100644 --- a/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs +++ b/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs @@ -580,7 +580,8 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common.Cmdlet throw new ArgumentException(Resources.DefaultStorageCredentialsNotFound); } - AzureStorageContext context = new AzureStorageContext(account, StorageAccountName); + AzureStorageContext context = new AzureStorageContext(account, StorageAccountName, DefaultContext, WriteDebug); + WriteObject(context); } } diff --git a/src/Storage/Storage/Common/StorageCloudCmdletBase.cs b/src/Storage/Storage/Common/StorageCloudCmdletBase.cs index f6dd72d377..6823d50dc0 100644 --- a/src/Storage/Storage/Common/StorageCloudCmdletBase.cs +++ b/src/Storage/Storage/Common/StorageCloudCmdletBase.cs @@ -281,7 +281,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common var context = inContext as AzureStorageContext; if (context == null && inContext != null) { - context = new AzureStorageContext(inContext.GetCloudStorageAccount()); + context = new AzureStorageContext(inContext.GetCloudStorageAccount(), null, DefaultContext, WriteDebug); } if (context != null) @@ -313,7 +313,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common } //Set the storage context and use it in pipeline - context = new AzureStorageContext(account); + context = new AzureStorageContext(account, null, DefaultContext, WriteDebug); } return context; diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs new file mode 100644 index 0000000000..2daf53cfd8 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs @@ -0,0 +1,134 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using Microsoft.Azure.Storage; + using Microsoft.Azure.Storage.Blob; + using System; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; + using System.Collections.Generic; + using global::Azure.Storage.Files.DataLake; + using global::Azure.Storage.Files.DataLake.Models; + using global::Azure; + + /// + /// list azure blobs in specified azure FileSystem + /// + [Cmdlet("Get", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2ChildItem"),OutputType(typeof(AzureDataLakeGen2Item))] + public class GetAzDataLakeGen2ChildItemCommand : StorageCloudBlobCmdletBase + { + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name")] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Position = 1, Mandatory = false, HelpMessage = + "The path in the specified FileSystem that should be retrieved. Can be a directory " + + "In the format 'directory1/directory2/'")] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Alias("FetchPermission")] + [Parameter(Mandatory = false, HelpMessage = "Fetch the datalake item properties and ACL.")] + [ValidateNotNullOrEmpty] + public SwitchParameter FetchProperty{ get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Indicates if will recursively get the Child Item. The default is false.")] + [ValidateNotNullOrEmpty] + public SwitchParameter Recurse { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "The max count of the blobs that can return.")] + public int? MaxCount + { + get { return InternalMaxCount; } + set + { + if (value.Value <= 0) + { + InternalMaxCount = int.MaxValue; + } + else + { + InternalMaxCount = value.Value; + } + } + } + + private int InternalMaxCount = int.MaxValue; + + [Parameter(Mandatory = false, HelpMessage = "Continuation Token.")] + public string ContinuationToken { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Run cmdlet in the background")] + public SwitchParameter AsJob { get; set; } + + [Alias("UserPrincipalName")] + [Parameter(Mandatory = false, HelpMessage = "If speicify this parameter, the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. " + + "If not speicify this parameter, the values will be returned as Azure Active Directory Object IDs. Note that group and application Object IDs are not translated because they do not have unique friendly names.")] + public SwitchParameter OutputUserPrincipalName { get; set; } + + // Overwrite the useless parameter + public override int? ConcurrentTaskCount { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the GetAzDataLakeGen2ChildItemCommand class. + /// + public GetAzDataLakeGen2ChildItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the GetAzDataLakeGen2ChildItemCommand class. + /// + /// IStorageBlobManagement channel + public GetAzDataLakeGen2ChildItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + /// + /// Execute command + /// + [PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + + BlobRequestOptions requestOptions = RequestOptions; + bool useFlatBlobListing = this.Recurse.IsPresent ? true : false; + + IEnumerator> enumerator = fileSystem.GetPaths(this.Path, this.Recurse, this.OutputUserPrincipalName.IsPresent) + .AsPages(this.ContinuationToken, this.MaxCount) + .GetEnumerator(); + + Page page; + enumerator.MoveNext(); + page = enumerator.Current; + foreach (PathItem item in page.Values) + { + WriteDataLakeGen2Item(localChannel, item, fileSystem, page.ContinuationToken, this.FetchProperty.IsPresent); + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2FileContent.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2FileContent.cs new file mode 100644 index 0000000000..d393cca732 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2FileContent.cs @@ -0,0 +1,282 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Microsoft.WindowsAzure.Commands.Common; +using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; +using Microsoft.WindowsAzure.Commands.Storage.Common; +using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; +using Microsoft.WindowsAzure.Commands.Utilities.Common; +using Microsoft.Azure.Storage; +using Microsoft.Azure.Storage.Blob; +using Microsoft.Azure.Storage.DataMovement; +using System; +using System.IO; +using System.Management.Automation; +using System.Security.Permissions; +using System.Threading.Tasks; +using Azure.Storage.Files.DataLake; + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + [Cmdlet("Get", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2ItemContent", SupportsShouldProcess = true, DefaultParameterSetName = ManualParameterSet),OutputType(typeof(AzureDataLakeGen2Item))] + public class GetAzDataLakeGen2ItemContentCommand : StorageDataMovementCmdletBase + { + /// + /// manually set the name parameter + /// + private const string ManualParameterSet = "ReceiveManual"; + + /// + /// blob pipeline + /// + private const string BlobParameterSet = "ItemPipeline"; + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Position = 1, Mandatory = true, HelpMessage = + "The path in the specified FileSystem that should be get content from. Must be a file." + + "In the format 'directory/file.txt'", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Azure Datalake Gen2 Item Object to download.", + ValueFromPipeline = true, ParameterSetName = BlobParameterSet)] + [ValidateNotNull] + public AzureDataLakeGen2Item InputObject { get; set; } + + [Parameter(HelpMessage = "Destination local file path.")] + public string Destination + { + get { return FileName; } + set { FileName = value; } + } + public string FileName = String.Empty; + + [Parameter(HelpMessage = "check the md5sum")] + public SwitchParameter CheckMd5 + { + get { return checkMd5; } + set { checkMd5 = value; } + } + + private bool checkMd5; + + private BlobToFileSystemNameResolver fileNameResolver; + + private DataLakeFileClient fileClient; + + // Overwrite the useless parameter + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the GetAzDataLakeGen2ItemContentCommand class. + /// + public GetAzDataLakeGen2ItemContentCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the GetAzDataLakeGen2ItemContentCommand class. + /// + /// IStorageBlobManagement channel + public GetAzDataLakeGen2ItemContentCommand(IStorageBlobManagement channel) + { + Channel = channel; + fileNameResolver = new BlobToFileSystemNameResolver(() => NameUtil.WindowsMaxFileNameLength); + } + + /// + /// Download blob to local file + /// + /// Source blob object + /// Destination file path + internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, string filePath) + { + string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath); + string status = Resources.PrepareDownloadingBlob; + ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); + DataMovementUserData data = new DataMovementUserData() + { + Data = blob, + TaskId = taskId, + Channel = localChannel, + Record = pr, + TotalSize = blob.Properties.Length + }; + + await DataMovementTransferHelper.DoTransfer(() => + { + return this.TransferManager.DownloadAsync(blob, filePath, + new DownloadOptions() + { + DisableContentMD5Validation = !this.checkMd5 + }, + this.GetTransferContext(data), + this.CmdletCancellationToken); + }, + data.Record, + this.OutputStream).ConfigureAwait(false); + + //this.WriteCloudBlobObject(data.TaskId, data.Channel, blob); + WriteDataLakeGen2Item(localChannel, fileClient, taskId: data.TaskId); + } + + /// + /// get blob content + /// + /// source CloudBlob object + /// destination file path + /// whether the source FileSystem validated + /// the downloaded blob object + internal void GetBlobContent(CloudBlob blob, string fileName, bool isValidBlob = false) + { + if (null == blob) + { + throw new ArgumentNullException(typeof(CloudBlob).Name, String.Format(Resources.ObjectCannotBeNull, typeof(CloudBlob).Name)); + } + + ValidateBlobType(blob); + + string filePath = GetFullReceiveFilePath(fileName, blob.Name); + + if (!isValidBlob) + { + ValidatePipelineCloudBlob(blob); + } + + //create the destination directory if not exists. + String dirPath = System.IO.Path.GetDirectoryName(filePath); + + if (!Directory.Exists(dirPath)) + { + Directory.CreateDirectory(dirPath); + } + + IStorageBlobManagement localChannel = Channel; + + Func taskGenerator = (taskId) => DownloadBlob(taskId, localChannel, blob, filePath); + RunTask(taskGenerator); + } + + /// + /// get full file path according to the specified file name + /// + /// File name + /// Source blob name + /// full file path if file path is valid, otherwise throw an exception + internal string GetFullReceiveFilePath(string fileName, string blobName) + { + String filePath = fileName; + fileName = System.IO.Path.GetFileName(filePath); + String dirPath = System.IO.Path.GetDirectoryName(filePath); + + if (!String.IsNullOrEmpty(dirPath) && !Directory.Exists(dirPath)) + { + throw new ArgumentException(String.Format(Resources.DirectoryNotExists, dirPath)); + } + + if (string.IsNullOrEmpty(fileName) || Directory.Exists(filePath)) + { + fileName = fileNameResolver.ResolveFileName(blobName, null); + filePath = System.IO.Path.Combine(filePath, fileName); + } + + fileName = System.IO.Path.GetFileName(filePath); + + if (!NameUtil.IsValidFileName(fileName)) + { + throw new ArgumentException(String.Format(Resources.InvalidFileName, fileName)); + } + + //there is no need to check the read/write permission on the specified file path, the datamovement libraray will do that + + return filePath; + } + + protected override void ProcessRecord() + { + try + { + FileName = GetUnresolvedProviderPathFromPSPath(FileName); + Validate.ValidateInternetConnection(); + InitChannelCurrentSubscription(); + this.ExecuteSynchronouslyOrAsJob(); + } + catch (Exception ex) when (!IsTerminatingError(ex)) + { + WriteExceptionError(ex); + } + } + + + /// + /// execute command + /// + [PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + BlobRequestOptions requestOptions = RequestOptions; + if (AsJob.IsPresent) + { + DoBeginProcessing(); + } + + CloudBlockBlob blob = null; + if (ParameterSetName == ManualParameterSet) + { + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + DataLakeDirectoryClient dirClient; + if (GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient)) + { + throw new ArgumentException(String.Format("The input FileSystem '{0}', path '{1}' point to a Directory, can't download it.", this.FileSystem, this.Path)); + } + + CloudBlobContainer container = GetCloudBlobContainerByName(Channel, this.FileSystem).ConfigureAwait(false).GetAwaiter().GetResult(); + blob = container.GetBlockBlobReference(this.Path); + } + else //BlobParameterSet + { + if (!InputObject.IsDirectory) + { + if (Channel.StorageContext.StorageAccount.Credentials.IsSAS) + { + // For SAS, the Uri already contains the sas token, so can't repeatedly inout the credential + blob = new CloudBlockBlob(InputObject.File.Uri); + } + else + { + blob = new CloudBlockBlob(InputObject.File.Uri, Channel.StorageContext.StorageAccount.Credentials); + } + fileClient = InputObject.File; + } + else + { + throw new ArgumentException(String.Format("The InputObject is a Directory, which don't have content to get.")); + } + } + + GetBlobContent(blob, FileName, true); + + if (AsJob.IsPresent) + { + DoEndProcessing(); + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2Item.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2Item.cs new file mode 100644 index 0000000000..705a1b329a --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2Item.cs @@ -0,0 +1,86 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using System.Management.Automation; + using System.Security.Permissions; + using global::Azure.Storage.Files.DataLake; + + /// + /// list azure blobs in specified azure FileSystem + /// + [Cmdlet("Get", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2Item"), OutputType(typeof(AzureDataLakeGen2Item))] + public class GetDataLakeGen2ItemCommand : StorageCloudBlobCmdletBase + { + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name")] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Mandatory = false, HelpMessage = + "The path in the specified FileSystem that should be retrieved. Can be a file or directory " + + "In the format 'directory/file.txt' or 'directory1/directory2/'. Not specify this parameter to get the root directory of the Filesystem.")] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + // Overwrite the useless parameter + public override int? ConcurrentTaskCount { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the GetDataLakeGen2ItemCommand class. + /// + public GetDataLakeGen2ItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the GetDataLakeGen2ItemCommand class. + /// + /// IStorageBlobManagement channel + public GetDataLakeGen2ItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + /// + /// Execute command + /// + [PermissionSet(SecurityAction.Demand, Name = "FullTrust")] + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + + DataLakeFileClient fileClient; + DataLakeDirectoryClient dirClient; + if (GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient)) + { + // Directory + WriteDataLakeGen2Item(localChannel, dirClient); + } + else + { + //File + WriteDataLakeGen2Item(Channel, fileClient); + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/MoveAzDataLakeGen2Item.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/MoveAzDataLakeGen2Item.cs new file mode 100644 index 0000000000..3072fb3b71 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/MoveAzDataLakeGen2Item.cs @@ -0,0 +1,173 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using Microsoft.Azure.Storage.Blob; + using System; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; + using System.Collections; + using System.Collections.Generic; + using global::Azure.Storage.Files.DataLake; + using global::Azure; + + /// + /// create a new azure FileSystem + /// + [Cmdlet("Move", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2Item", DefaultParameterSetName = ManualParameterSet, SupportsShouldProcess = true),OutputType(typeof(AzureDataLakeGen2Item))] + public class MoveAzDataLakeGen2ItemCommand : StorageCloudBlobCmdletBase + { + /// + /// manually set the name parameter + /// + private const string ManualParameterSet = "ReceiveManual"; + + /// + /// Blob or BlobDir pipeline + /// + private const string BlobParameterSet = "ItemPipeline"; + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Position = 1, Mandatory = true, HelpMessage = + "The path in the specified FileSystem that should be move from. Can be a file or directory. " + + "In the format 'directory/file.txt' or 'directory1/directory2/'", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Azure Datalake Gen2 Item Object to move from.", + ValueFromPipeline = true, ParameterSetName = BlobParameterSet)] + [ValidateNotNull] + public AzureDataLakeGen2Item InputObject { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Dest FileSystem name")] + [ValidateNotNullOrEmpty] + public string DestFileSystem { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Dest item path")] + [ValidateNotNullOrEmpty] + public string DestPath { get; set; } + + [Parameter(HelpMessage = "Force to over write the destination.")] + public SwitchParameter Force { get; set; } + + // Overwrite the useless parameter + public override int? ConcurrentTaskCount { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the MoveAzDataLakeGen2ItemCommand class. + /// + public MoveAzDataLakeGen2ItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the MoveAzDataLakeGen2ItemCommand class. + /// + /// IStorageBlobManagement channel + public MoveAzDataLakeGen2ItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + /// + /// execute command + /// + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + BlobRequestOptions requestOptions = RequestOptions; + + bool foundAFolder = false; + DataLakeFileClient srcBlob = null; + DataLakeDirectoryClient srcBlobDir = null; + if (ParameterSetName == ManualParameterSet) + { + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir); + } + else //BlobParameterSet + { + if (!InputObject.IsDirectory) + { + srcBlob = InputObject.File; + } + else + { + srcBlobDir = InputObject.Directory; + foundAFolder = true; + } + } + + if (foundAFolder) + { + if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: ")) + { + // check dest exist + bool destExist = true; + DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); + DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath); + try + { + destBlobDir.GetProperties(); + } + catch (RequestFailedException e) when (e.Status == 404) + { + destExist = false; + } + + if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), "")) + { + destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value; + WriteDataLakeGen2Item(localChannel, destBlobDir); + } + } + } + else + { + if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: ")) + { + // check dest exist + bool destExist = true; + DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); + DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath); + try + { + destFile.GetProperties(); + } + catch (RequestFailedException e) when (e.Status == 404) + { + destExist = false; + } + + if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), "")) + { + destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value; + WriteDataLakeGen2Item(localChannel, destFile); + } + } + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2ACLObject.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2ACLObject.cs new file mode 100644 index 0000000000..43853a3d44 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2ACLObject.cs @@ -0,0 +1,89 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +//using Microsoft.Azure.Storage.Blob; +using Microsoft.WindowsAzure.Commands.Common; +using Microsoft.WindowsAzure.Commands.Storage; +using Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel; +using System.Collections.Generic; +using System.Globalization; +using System.Management.Automation; +using global::Azure.Storage.Files.DataLake; +using global::Azure; +using global::Azure.Storage.Files.DataLake.Models; + +namespace Microsoft.Azure.Commands.Management.Storage +{ + [Cmdlet("Set", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2ItemAclObject"), OutputType(typeof(PSPathAccessControlEntry))] + [Alias("New-" + Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2ItemAclObject")] + public class SetAzDataLakeGen2ItemAclObjectCommand : AzureDataCmdlet + { + [Parameter(Mandatory = false, HelpMessage = "The user or group identifier. It is omitted for entries of AccessControlType \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group.")] + [ValidateNotNullOrEmpty] + public string EntityId { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Set this parameter to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL.")] + [ValidateNotNullOrEmpty] + public SwitchParameter DefaultScope { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "There are four types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries.")] + [ValidateNotNullOrEmpty] + [ValidateSet(PSAccessControlType.User, + PSAccessControlType.Group, + PSAccessControlType.Mask, + PSAccessControlType.Other, + IgnoreCase = true)] + public AccessControlType AccessControlType; + + [Parameter(Mandatory = true, HelpMessage = "The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied.")] + [ValidatePattern("[r-][w-][x-]")] + public string Permission { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "If input the PSPathAccessControlEntry[] object, will add the new ACL entry as a new element of the input PSPathAccessControlEntry[] object. If an ACL entry when same AccessControlType, EntityId, DefaultScope exist, will update permission of it.")] + [ValidateNotNullOrEmpty] + public PSPathAccessControlEntry[] InputObject { get; set; } + + public override void ExecuteCmdlet() + { + base.ExecuteCmdlet(); + + List psacls = new List(); + if (InputObject !=null) + { + psacls = new List(this.InputObject); + } + + // Remove the ACL entry to add if already exist, to avoid duplicated entries + PSPathAccessControlEntry entryToRemove = null; + foreach (PSPathAccessControlEntry entry in psacls) + { + if (entry.DefaultScope == this.DefaultScope.IsPresent + && entry.AccessControlType == this.AccessControlType + && entry.EntityId == this.EntityId) + { + entryToRemove = entry; + } + } + if (entryToRemove != null) + { + psacls.Remove(entryToRemove); + } + + PSPathAccessControlEntry psacl = new PSPathAccessControlEntry(this.AccessControlType, PathAccessControlExtensions.ParseSymbolicRolePermissions(this.Permission), this.DefaultScope, this.EntityId); + psacls.Add(psacl); + + WriteObject(psacls.ToArray(), true); + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2Item.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2Item.cs new file mode 100644 index 0000000000..4a86d2cac0 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/NewAzDataLakeGen2Item.cs @@ -0,0 +1,390 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Common; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using Microsoft.Azure.Storage.Blob; + using System; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; + using System.Collections; + using System.Collections.Generic; + using Microsoft.Azure.Storage.DataMovement; + using System.IO; + using Microsoft.WindowsAzure.Commands.Common; + using Microsoft.WindowsAzure.Commands.Utilities.Common; + using global::Azure.Storage.Files.DataLake; + using global::Azure.Storage.Files.DataLake.Models; + using DataLakeModels = global::Azure.Storage.Files.DataLake.Models; + using System.Globalization; + + /// + /// create a new DataLakeGen2 Item + /// + [Cmdlet("New", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2Item", SupportsShouldProcess = true, DefaultParameterSetName = FileParameterSet), OutputType(typeof(AzureDataLakeGen2Item))] + public class NewAzDataLakeGen2ItemCommand : StorageDataMovementCmdletBase + { + /// + /// Create a Directory parameter + /// + private const string DirectoryParameterSet = "Directory"; + + /// + /// Create a file parameter + /// + private const string FileParameterSet = "File"; + + private const string defaultFilePermission = "rw-rw-rw-"; + private const string defaultUmask = "----w-rwx"; + + private DataLakeFileSystemClient fileSystem; + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name")] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Position = 1, Mandatory = true, HelpMessage = + "The path in the specified FileSystem that should be create. Can be a file or directory " + + "In the format 'directory/file.txt' or 'directory1/directory2/'")] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Indicates that this new item is a directory and not a file.", ParameterSetName = DirectoryParameterSet)] + public SwitchParameter Directory { get; set; } + + [Parameter(ValueFromPipeline = true, Mandatory = true, HelpMessage = "Specify the local source file path which will be upload to a Datalake Gen2 file.", ParameterSetName = FileParameterSet)] + [ValidateNotNullOrEmpty] + public string Source + { + get { return FileName; } + set { FileName = value; } + } + + private string FileName = String.Empty; + + [Parameter(Mandatory = false, + HelpMessage = "When creating New Item and the parent directory does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. Symbolic (rwxrw-rw-) is supported.")] + [ValidateNotNullOrEmpty] + [ValidatePattern("([r-][w-][x-]){3}")] + public string Umask { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. Symbolic (rwxrw-rw-) is supported. ")] + [ValidateNotNullOrEmpty] + [ValidatePattern("([r-][w-][x-]){3}")] + public string Permission { get; set; } + + + [Parameter(HelpMessage = "Specifies properties for the created directory or file. "+ + "The supported properties for file are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentMD5, ContentType." + + "The supported properties for directory are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage.", + Mandatory = false)] + public Hashtable Property + { + get + { + return BlobProperties; + } + + set + { + BlobProperties = value; + } + } + private Hashtable BlobProperties = null; + + [Parameter(HelpMessage = "Specifies metadata for the created directory or file.", Mandatory = false)] + public Hashtable Metadata + { + get + { + return BlobMetadata; + } + + set + { + BlobMetadata = value; + } + } + private Hashtable BlobMetadata = null; + + // Overwrite the useless parameter + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the NewAzDataLakeGen2ItemCommand class. + /// + public NewAzDataLakeGen2ItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the NewAzDataLakeGen2ItemCommand class. + /// + /// IStorageBlobManagement channel + public NewAzDataLakeGen2ItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + protected override void ProcessRecord() + { + try + { + ResolvedFileName = this.GetUnresolvedProviderPathFromPSPath(string.IsNullOrWhiteSpace(this.FileName) ? "." : this.FileName); + Validate.ValidateInternetConnection(); + InitChannelCurrentSubscription(); + this.ExecuteSynchronouslyOrAsJob(); + } + catch (Exception ex) when (!IsTerminatingError(ex)) + { + WriteExceptionError(ex); + } + } + + /// + /// execute command + /// + public override void ExecuteCmdlet() + { + if (AsJob.IsPresent) + { + DoBeginProcessing(); + } + + IStorageBlobManagement localChannel = Channel; + fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + + if (this.Directory.IsPresent) + { + DataLakeDirectoryClient dirClient = fileSystem.GetDirectoryClient(this.Path); + if (ShouldProcess(GetDataLakeItemUriWithoutSas(dirClient), "Create Directory: ")) + { + if (dirClient.Exists()) + { + throw new ResourceAlreadyExistException(String.Format("Folder '{0}' already exists.", GetDataLakeItemUriWithoutSas(dirClient))); + } + DataLakeModels.PathPermissions pathPermissions = null; + if (this.Permission != null) + { + pathPermissions = DataLakeModels.PathPermissions.ParseSymbolicPermissions(this.Permission); + } + + // Set BlobDir Properties and MetaData + PathHttpHeaders pathHttpHeaders = SetDatalakegen2ItemProperties(dirClient, BlobProperties, setToServer: false); + IDictionary metadata = SetDatalakegen2ItemMetaData(dirClient, BlobMetadata, setToServer: false); + + dirClient.Create(pathHttpHeaders, + metadata, + this.Permission, + this.Umask != null ? DataLakeModels.PathPermissions.ParseSymbolicPermissions(this.Umask).ToOctalPermissions() : null); + + WriteDataLakeGen2Item(localChannel, dirClient); + } + } + else //create File + { + DataLakeFileClient fileClient = fileSystem.GetFileClient(this.Path); + if (ShouldProcess(GetDataLakeItemUriWithoutSas(fileClient), "Create File: ")) + { + // Use SDK to upload directly when use SAS credential, and need set permission, since set permission after upload will fail with SAS + if (Channel.StorageContext.StorageAccount.Credentials.IsSAS + && (!string.IsNullOrEmpty(this.Permission) || !string.IsNullOrEmpty(this.Umask))) + { + Func taskGenerator = (taskId) => UploadDataLakeFile(taskId, fileClient, ResolvedFileName); + RunTask(taskGenerator); + } + else + { + CloudBlobContainer container = Channel.GetContainerReference(this.FileSystem); + CloudBlockBlob blob = container.GetBlockBlobReference(this.Path); + + Func taskGenerator = (taskId) => Upload2Blob(taskId, Channel, ResolvedFileName, blob); + RunTask(taskGenerator); + } + } + } + + + if (AsJob.IsPresent) + { + DoEndProcessing(); + } + } + + /// + /// Upload File with Datalake API + /// + internal virtual async Task UploadDataLakeFile(long taskId, DataLakeFileClient fileClient, string filePath) + { + if (this.Force.IsPresent || !fileClient.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, GetDataLakeItemUriWithoutSas(fileClient)), null)) + { + + // Set Item Properties and MetaData + PathHttpHeaders pathHttpHeaders = SetDatalakegen2ItemProperties(fileClient, BlobProperties, setToServer: false); + IDictionary metadata = SetDatalakegen2ItemMetaData(fileClient, BlobMetadata, setToServer: false); + + fileClient.Create(pathHttpHeaders, + metadata, + this.Permission, + this.Umask != null ? DataLakeModels.PathPermissions.ParseSymbolicPermissions(this.Umask).ToOctalPermissions() : null); + + long fileSize = new FileInfo(ResolvedFileName).Length; + string activity = String.Format(Resources.SendAzureBlobActivity, this.Source, this.Path, this.FileSystem); + string status = Resources.PrepareUploadingBlob; + ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); + IProgress progressHandler = new Progress((finishedBytes) => + { + if (pr != null) + { + // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. + pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize); + pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete); + this.OutputStream.WriteProgress(pr); + } + }); + + using (FileStream stream = File.OpenRead(ResolvedFileName)) + { + await fileClient.AppendAsync(stream, 0, progressHandler: progressHandler, cancellationToken: CmdletCancellationToken).ConfigureAwait(false); + } + WriteDataLakeGen2Item(Channel, fileClient, taskId: taskId); + } + + } + + /// + /// Set blob content + /// + /// Dest CloudBlob object + /// source local file path + /// whether the source FileSystem validated + /// the uploaded blob object + internal void SetBlobContent(CloudBlockBlob blob, string fileName, bool isValidBlob = false) + { + if (!isValidBlob) + { + ValidatePipelineCloudBlob(blob); + } + + //UploadBlob(taskId, localChannel, blob, filePath); + Func taskGenerator = (taskId) => Upload2Blob(taskId, Channel, fileName, blob); + RunTask(taskGenerator); + } + + /// + /// upload file to azure blob + /// + /// Task id + /// local file path + /// destination azure blob object + internal virtual async Task Upload2Blob(long taskId, IStorageBlobManagement localChannel, string filePath, CloudBlob blob) + { + string activity = String.Format(Resources.SendAzureBlobActivity, filePath, blob.Name, blob.Container.Name); + string status = Resources.PrepareUploadingBlob; + ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); + + FileInfo fileInfo = new FileInfo(filePath); + + DataMovementUserData data = new DataMovementUserData() + { + Data = blob, + TaskId = taskId, + Channel = localChannel, + Record = pr, + TotalSize = fileInfo.Length + }; + + SingleTransferContext transferContext = this.GetTransferContext(data); + +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + transferContext.SetAttributesCallbackAsync = async (destination) => +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + { + CloudBlob destBlob = destination as CloudBlob; + SetAzureBlobContentCommand.SetBlobProperties(destBlob, this.BlobProperties); + SetAzureBlobContentCommand.SetBlobMeta(destBlob, this.BlobMetadata); + }; + + await DataMovementTransferHelper.DoTransfer(() => + { + return this.TransferManager.UploadAsync(filePath, + blob, + null, + transferContext, + this.CmdletCancellationToken); + }, + data.Record, + this.OutputStream).ConfigureAwait(false); + + // Set blob permission with umask, since create Blob API still not support them + SetBlobPermissionWithUMask((CloudBlockBlob)blob, this.Permission, this.Umask); + + WriteDataLakeGen2Item(localChannel, fileSystem.GetFileClient(blob.Name), taskId: data.TaskId); + } + + /// + /// Set block blob permission with Umask after blob is created + /// Add this since XSCL don't have interface to set permission with umask in create blob + /// Won't set when both permission and umask are null/empty, and use server default behavior. + /// + /// the blob object to set permission with umask + /// permission string to set, in format "rwxrwxrwx", default value is "rwxrwxrwx" + /// umask string to set, in format "rwxrwxrwx", default value is "----w-rwx" + protected void SetBlobPermissionWithUMask(CloudBlockBlob blob, string permission, String umask) + { + // Don't need set permission when both input permission and umask are null + if (string.IsNullOrEmpty(permission) && string.IsNullOrEmpty(umask)) + { + return; + } + + //Set the default value if one of permission or Umask is null + // Confirmed with feature team: + // Default permission for files is 666 (default permission for directory is 777, this function is for file so use 666) + // default umask is 027 + if (string.IsNullOrEmpty(permission)) + { + permission = defaultFilePermission; + } + if (string.IsNullOrEmpty(umask)) + { + umask = defaultUmask; + } + + // Get the permission value to set, from input Permission and Umask + // The permission and umask string format is already checked with parameter ValidatePattern + string blobPermission = string.Empty; + for (int i = 0; i < permission.Length; i++) + { + if (umask[i] != '-') + { + blobPermission += '-'; + } + else + { + blobPermission += permission[i]; + } + } + + //Set permission to blob + fileSystem.GetFileClient(blob.Name).SetPermissions(DataLakeModels.PathPermissions.ParseSymbolicPermissions(blobPermission)); + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/RemoveAzDataLakeGen2Item.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/RemoveAzDataLakeGen2Item.cs new file mode 100644 index 0000000000..a4a274ce74 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/RemoveAzDataLakeGen2Item.cs @@ -0,0 +1,151 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using Microsoft.Azure.Storage.Blob; + using System; + using System.Management.Automation; + using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; + using global::Azure.Storage.Files.DataLake; + + /// + /// remove specified azure FileSystem + /// + [Cmdlet("Remove", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2Item", DefaultParameterSetName = ManualParameterSet, SupportsShouldProcess = true),OutputType(typeof(Boolean))] + public class RemoveAzDataLakeGen2ItemCommand : StorageCloudBlobCmdletBase + { + /// + /// manually set the name parameter + /// + private const string ManualParameterSet = "ReceiveManual"; + + /// + /// Blob or BlobDir pipeline + /// + private const string BlobParameterSet = "ItemPipeline"; + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Position = 1, Mandatory = true, HelpMessage = + "The path in the specified FileSystem that should be removed. Can be a file or directory " + + "In the format 'directory/file.txt' or 'directory1/directory2/'", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Azure Datalake Gen2 Item Object to remove.", + ValueFromPipeline = true, ParameterSetName = BlobParameterSet)] + [ValidateNotNull] + public AzureDataLakeGen2Item InputObject { get; set; } + + [Parameter(HelpMessage = "Force to remove the Item without any prompt.")] + public SwitchParameter Force + { + get { return force; } + set { force = value; } + } + private bool force; + + [Parameter(Mandatory = false, HelpMessage = "Run cmdlet in the background")] + public SwitchParameter AsJob { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Return whether the specified FileSystem is successfully removed")] + public SwitchParameter PassThru { get; set; } + + // Overwrite the useless parameter + public override int? ConcurrentTaskCount { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + /// + /// Initializes a new instance of the RemoveAzDataLakeGen2ItemCommand class. + /// + public RemoveAzDataLakeGen2ItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the RemoveAzDataLakeGen2ItemCommand class. + /// + /// IStorageBlobManagement channel + public RemoveAzDataLakeGen2ItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + /// + /// Cmdlet begin processing + /// + protected override void BeginProcessing() + { + base.BeginProcessing(); + OutputStream.ConfirmWriter = (s1, s2, s3) => ShouldContinue(s2, s3); + } + + /// + /// execute command + /// + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + BlobRequestOptions requestOptions = RequestOptions; + + bool foundAFolder = false; + + DataLakeFileClient fileClient = null; + DataLakeDirectoryClient dirClient = null; + if (ParameterSetName == ManualParameterSet) + { + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient); + } + else //BlobParameterSet + { + if (!InputObject.IsDirectory) + { + fileClient = InputObject.File; + } + else + { + dirClient = InputObject.Directory; + foundAFolder = true; + } + } + + if (foundAFolder) + { + if (force || ShouldContinue(string.Format("Remove Directory: {0}", GetDataLakeItemUriWithoutSas(dirClient)), "")) + { + dirClient.Delete(true); + } + } + else + { + if (force || ShouldContinue(string.Format("Remove File: {0}", GetDataLakeItemUriWithoutSas(fileClient)), "")) + { + fileClient.Delete(); + } + } + + if (PassThru) + { + WriteObject(true); + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/UpdateAzDataLakeGen2Item.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/UpdateAzDataLakeGen2Item.cs new file mode 100644 index 0000000000..8d15686349 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/UpdateAzDataLakeGen2Item.cs @@ -0,0 +1,218 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet +{ + using Commands.Common.Storage.ResourceModel; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using System.Management.Automation; + using System.Collections; + using Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel; + using global::Azure.Storage.Files.DataLake; + using global::Azure.Storage.Files.DataLake.Models; + + /// + /// create a new azure FileSystem + /// + [Cmdlet("Update", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "DataLakeGen2Item", DefaultParameterSetName = ManualParameterSet, SupportsShouldProcess = true),OutputType(typeof(AzureDataLakeGen2Item))] + public class SetAzDataLakeGen2ItemCommand : StorageCloudBlobCmdletBase + { + /// + /// manually set the name parameter + /// + private const string ManualParameterSet = "ReceiveManual"; + + /// + /// Blob or BlobDir pipeline + /// + private const string BlobParameterSet = "ItemPipeline"; + + [Parameter(ValueFromPipeline = true, Position = 0, Mandatory = true, HelpMessage = "FileSystem name", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string FileSystem { get; set; } + + [Parameter(ValueFromPipeline = true, Mandatory = false, HelpMessage = + "The path in the specified FileSystem that should be updated. Can be a file or directory " + + "In the format 'directory/file.txt' or 'directory1/directory2/'. Not specify this parameter will update the root directory of the Filesystem.", ParameterSetName = ManualParameterSet)] + [ValidateNotNullOrEmpty] + public string Path { get; set; } + + [Parameter(Mandatory = true, HelpMessage = "Azure Datalake Gen2 Item Object to update", + ValueFromPipeline = true, ParameterSetName = BlobParameterSet)] + [ValidateNotNull] + public AzureDataLakeGen2Item InputObject { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. Symbolic (rwxrw-rw-) is supported. Invalid in conjunction with ACL.")] + [ValidateNotNullOrEmpty] + [ValidatePattern("([r-][w-][x-]){3}")] + public string Permission { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Sets the owner of the item.")] + [ValidateNotNullOrEmpty] + public string Owner { get; set; } + + [Parameter(Mandatory = false, HelpMessage = "Sets the owning group of the item.")] + [ValidateNotNullOrEmpty] + public string Group { get; set; } + + [Parameter(HelpMessage = "Specifies properties for the directory or file. " + + "The supported properties for file are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentMD5, ContentType." + + "The supported properties for directory are: CacheControl, ContentDisposition, ContentEncoding, ContentLanguage.", + Mandatory = false)] + public Hashtable Property + { + get + { + return BlobProperties; + } + + set + { + BlobProperties = value; + } + } + private Hashtable BlobProperties = null; + + [Parameter(HelpMessage = "Specifies metadata for the directory or file.", + Mandatory = false)] + public Hashtable Metadata + { + get + { + return BlobMetadata; + } + + set + { + BlobMetadata = value; + } + } + private Hashtable BlobMetadata = null; + + [Parameter(HelpMessage = "Sets POSIX access control rights on files and directories. Create this object with New-AzDataLakeGen2ItemAclObject.", Mandatory = false)] + [ValidateNotNullOrEmpty] + public PSPathAccessControlEntry[] Acl { get; set; } + + // Overwrite the useless parameter + public override int? ConcurrentTaskCount { get; set; } + public override int? ClientTimeoutPerRequest { get; set; } + public override int? ServerTimeoutPerRequest { get; set; } + + + /// + /// Initializes a new instance of the SetAzDataLakeGen2ItemCommand class. + /// + public SetAzDataLakeGen2ItemCommand() + : this(null) + { + } + + /// + /// Initializes a new instance of the SetAzDataLakeGen2ItemCommand class. + /// + /// IStorageBlobManagement channel + public SetAzDataLakeGen2ItemCommand(IStorageBlobManagement channel) + { + Channel = channel; + } + + /// + /// execute command + /// + public override void ExecuteCmdlet() + { + IStorageBlobManagement localChannel = Channel; + + bool foundAFolder = false; + + DataLakeFileClient fileClient = null; + DataLakeDirectoryClient dirClient = null; + if (ParameterSetName == ManualParameterSet) + { + DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); + foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient); + } + else //BlobParameterSet + { + if (!InputObject.IsDirectory) + { + fileClient = InputObject.File; + } + else + { + dirClient = InputObject.Directory; + foundAFolder = true; + } + } + + if (foundAFolder) + { + if (ShouldProcess(GetDataLakeItemUriWithoutSas(dirClient), "Update Directory: ")) + { + //Set Permission + if (this.Permission != null || this.Owner != null || this.Group != null) + { + //PathAccessControl originPathAccessControl = dirClient.GetAccessControl().Value; + dirClient.SetPermissions( + this.Permission != null ? PathPermissions.ParseSymbolicPermissions(this.Permission) : null, + this.Owner, + this.Group); + } + + //Set ACL + if (this.Acl != null) + { + dirClient.SetAccessControlList(PSPathAccessControlEntry.ParseAccessControls(this.Acl)); + } + + // Set Properties + SetDatalakegen2ItemProperties(dirClient, this.BlobProperties, setToServer: true); + + //Set MetaData + SetDatalakegen2ItemMetaData(dirClient, this.BlobMetadata, setToServer: true); + + WriteDataLakeGen2Item(localChannel, dirClient); + } + } + else + { + if (ShouldProcess(GetDataLakeItemUriWithoutSas(fileClient), "Update File: ")) + { + //Set Permission + if (this.Permission != null || this.Owner != null || this.Group != null) + { + fileClient.SetPermissions( + this.Permission != null ? PathPermissions.ParseSymbolicPermissions(this.Permission) : null, + this.Owner, + this.Group); + } + + //Set ACL + if (this.Acl != null) + { + fileClient.SetAccessControlList(PSPathAccessControlEntry.ParseAccessControls(this.Acl)); + } + + // Set Properties + SetDatalakegen2ItemProperties(fileClient, this.BlobProperties, setToServer: true); + + //Set MetaData + SetDatalakegen2ItemMetaData(fileClient, this.BlobMetadata, setToServer: true); + + WriteDataLakeGen2Item(localChannel, fileClient); + } + } + } + } +} diff --git a/src/Storage/Storage/DatalakeGen2/GEN1_GEN2_MAPPING.md b/src/Storage/Storage/DatalakeGen2/GEN1_GEN2_MAPPING.md new file mode 100644 index 0000000000..a91427e6b8 --- /dev/null +++ b/src/Storage/Storage/DatalakeGen2/GEN1_GEN2_MAPPING.md @@ -0,0 +1,136 @@ +

Mapping from ADLS Gen1 Cmdlets -> ADLS Gen2 Cmdlets

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ADLS Gen1 CmdletADLS Gen2 CmdletNotes
Add-AzDataLakeStoreItemContentN/A
Export-AzDataLakeStoreChildItemPropertyN/A
Export-AzDataLakeStoreItemN/A
Get-AzDataLakeStoreChildItemGet-AzDataLakeGen2ChildItemBy default Get-AzDataLakeGen2ChildItem will only list the first level child items. With “-Recurse” will list child items recursively.
Get-AzDataLakeStoreChildItemSummaryN/A
Get-AzDataLakeStoreItemGet-AzDataLakeGen2ItemThe output items of Get-AzDataLakeGen2Item has properties: Acl, Owner, Group, Permission.
Get-AzDataLakeStoreItemAclEntry
Get-AzDataLakeStoreItemOwner
Get-AzDataLakeStoreItemPermission
Get-AzDataLakeStoreItemContentGet-AzDataLakeGen2FileContentGet-AzDataLakeGen2FileContent will download File content to local file.
Import-AzDataLakeStoreItemN/A
Join-AzDataLakeStoreItemN/A
Move-AzDataLakeStoreItemMove-AzDataLakeGen2Item
New-AzDataLakeStoreItemNew-AzDataLakeGen2ItemCreate a File with New-AzDataLakeGen2Item, will upload the new File content from a local file.
Remove-AzDataLakeStoreItemRemove-AzDataLakeGen2Item
Set-AzDataLakeStoreItemOwnerUpdate-AzDataLakeGen2ItemUpdate-AzDataLakeGen2Item only update single item, not recursively. (If want to update recursively, list items recursively with Get-AzDataLakeStoreChildItem, then pipeline to Update-AzDataLakeGen2Item.)
Set-AzDataLakeStoreItemPermission
Set-AzDataLakeStoreItemAclTo update Acl with Update-AzDataLakeGen2Item, prepare the ACL object with cmdlet New-AzDataLakeGen2ItemAclObject.
Remove-AzDataLakeStoreItemAclN/A
Remove-AzDataLakeStoreItemAclEntryN/A
Set-AzDataLakeStoreItemAclEntryN/A
Set-AzDataLakeStoreItemExpiryN/A
Test-AzDataLakeStoreItemGet-AzDataLakeGen2ItemGet-AzDataLakeGen2Item will report error when get not exist item.
N/ANew-AzDatalakeGen2FileSystemNew cmdlets in Gen2, to manage File System. They are actually cmdlet alias of blob container cmdlets.
N/AGet-AzDatalakeGen2FileSystem
N/ARemove-AzDatalakeGen2FileSystem
N/ANew-AzDataLakeGen2ItemAclObject New cmdlet in Gen2, to prepare the Acl object, which will be used in cmdlet Update-AzDataLakeGen2Item.
\ No newline at end of file diff --git a/src/Storage/Storage/Model/ResourceModel/PSPathAccessControl.cs b/src/Storage/Storage/Model/ResourceModel/PSPathAccessControl.cs new file mode 100644 index 0000000000..02adb3224c --- /dev/null +++ b/src/Storage/Storage/Model/ResourceModel/PSPathAccessControl.cs @@ -0,0 +1,97 @@ +// ---------------------------------------------------------------------------------- +// +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using Azure.Storage.Files.DataLake.Models; + +namespace Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel +{ + /// + /// Wrapper class of Microsoft.Azure.Storage.Blob.PathAccessControlEntry + /// + public class PSPathAccessControlEntry + { + + public bool DefaultScope; + public AccessControlType AccessControlType; + public string EntityId; + public RolePermissions Permissions; + + public PSPathAccessControlEntry(AccessControlType accessControlType, RolePermissions permissions, bool defaultScope = false, string entityId = null) + { + this.DefaultScope = defaultScope; + this.AccessControlType = accessControlType; + this.EntityId = entityId; + this.Permissions = permissions; + } + public PSPathAccessControlEntry(PathAccessControlItem acl) + { + this.DefaultScope = acl.DefaultScope; + this.AccessControlType = acl.AccessControlType; + this.EntityId = acl.EntityId; + this.Permissions = acl.Permissions; + } + + public static List ParseAccessControls(PSPathAccessControlEntry[] psacls) + { + if (psacls == null || psacls.Count() == 0) + { + return null; + } + List acls = new List(); + foreach (PSPathAccessControlEntry psacl in psacls) + { + acls.Add(new PathAccessControlItem() + { + AccessControlType = psacl.AccessControlType, + Permissions = psacl.Permissions, + DefaultScope = psacl.DefaultScope, + EntityId = psacl.EntityId + }); + } + return acls; + } + + public static PSPathAccessControlEntry[] ParsePSPathAccessControlEntrys(IEnumerable acls) + { + if (acls == null || acls.Count() == 0) + { + return null; + } + List psacls = new List(); + foreach (PathAccessControlItem acl in acls) + { + psacls.Add(new PSPathAccessControlEntry(acl)); + } + return psacls.ToArray(); + } + + public string GetSymbolicRolePermissions() + { + return PathAccessControlExtensions.ToSymbolicRolePermissions(this.Permissions); + } + } + + /// + /// wrapper class of Microsoft.Azure.Storage.Blob.AccessControlType + /// + public static class PSAccessControlType + { + public const string User = "User"; + public const string Group = "Group"; + public const string Mask = "Mask"; + public const string Other = "Other"; + } +} diff --git a/src/Storage/Storage/Storage.csproj b/src/Storage/Storage/Storage.csproj index 1815c67748..9b3a077de3 100644 --- a/src/Storage/Storage/Storage.csproj +++ b/src/Storage/Storage/Storage.csproj @@ -1,4 +1,4 @@ - + Storage @@ -12,14 +12,24 @@ - - - - - + + + + + + + + + + + + + + + $(RepoSrc)lib\Microsoft.Azure.Storage.DataMovement.dll diff --git a/src/Storage/Storage/Storage.format.ps1xml b/src/Storage/Storage/Storage.format.ps1xml index c154eb08a6..6043deb84d 100644 --- a/src/Storage/Storage/Storage.format.ps1xml +++ b/src/Storage/Storage/Storage.format.ps1xml @@ -517,5 +517,111 @@ + + Microsoft.Azure.Storage.Blob.CloudBlobDirectory + + Microsoft.Azure.Storage.Blob.CloudBlobDirectory + + + $_.Container.Uri + + + + + + + Left + + + + Left + + + + Left + + + + Left + + + + Left + + + + + + + $_.Prefix + + + $_.PathProperties.Owner + + + $_.PathProperties.Group + + + $_.PathProperties.Permissions.ToSymbolicString() + + + $_.Properties.LastModified.UtcDateTime.ToString("u") + + + + + + + + Microsoft.Azure.Storage.Blob.PathPermissions + + Microsoft.Azure.Storage.Blob.PathPermissions + + + + + + Left + + + + Left + + + + Left + + + + Left + + + + Left + + + + + + + $_.Owner.ToSymbolicString() + + + $_.Group.ToSymbolicString() + + + $_.Other.ToSymbolicString() + + + StickyBit + + + ExtendedInfoInAcl + + + + + + diff --git a/src/Storage/Storage/Storage.generated.format.ps1xml b/src/Storage/Storage/Storage.generated.format.ps1xml index 846372c226..1644b31ec2 100644 --- a/src/Storage/Storage/Storage.generated.format.ps1xml +++ b/src/Storage/Storage/Storage.generated.format.ps1xml @@ -305,6 +305,89 @@ + + Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + + Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item + + + + + Left + + 20 + + + Left + + 12 + + + Left + + 15 + + + Left + + 20 + + + + Left + 12 + + + Left + + 20 + + + Left + + 20 + + + + + + + Left + Path + + + Left + IsDirectory + + + Left + if ($_.IsDirectory -eq $false) {$_.Length} + + + Left + $_.LastModified.UtcDateTime.ToString("u") + + + Left + $_.Permissions.ToSymbolicPermissions() + + + Left + Owner + + + Left + Group + + + + + + + if ($_.IsDirectory) {$_.Directory.FileSystemName} else {$_.File.FileSystemName} + + + Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSFileHandle @@ -384,5 +467,49 @@ + + Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSPathAccessControlEntry + + Microsoft.WindowsAzure.Commands.Storage.Model.ResourceModel.PSPathAccessControlEntry + + + + + + Left + + + + Left + + + + Left + + + + Left + + + + + + + DefaultScope + + + AccessControlType + + + EntityId + + + $_.GetSymbolicRolePermissions() + + + + + + \ No newline at end of file diff --git a/src/lib/Azure.Core/NetCoreAssemblies/Azure.Core.dll b/src/lib/Azure.Core/NetCoreAssemblies/Azure.Core.dll new file mode 100644 index 0000000000..e52829cd90 Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/Azure.Core.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/Microsoft.Bcl.AsyncInterfaces.dll b/src/lib/Azure.Core/NetCoreAssemblies/Microsoft.Bcl.AsyncInterfaces.dll new file mode 100644 index 0000000000..0a784888ba Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/Microsoft.Bcl.AsyncInterfaces.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/System.Numerics.Vectors.dll b/src/lib/Azure.Core/NetCoreAssemblies/System.Numerics.Vectors.dll new file mode 100644 index 0000000000..10205772c3 Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/System.Numerics.Vectors.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/System.Runtime.CompilerServices.Unsafe.dll b/src/lib/Azure.Core/NetCoreAssemblies/System.Runtime.CompilerServices.Unsafe.dll new file mode 100644 index 0000000000..be64036f12 Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/System.Runtime.CompilerServices.Unsafe.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Encodings.Web.dll b/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Encodings.Web.dll new file mode 100644 index 0000000000..cbf4938528 Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Encodings.Web.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Json.dll b/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Json.dll new file mode 100644 index 0000000000..0491dede3b Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/System.Text.Json.dll differ diff --git a/src/lib/Azure.Core/NetCoreAssemblies/System.Threading.Tasks.Extensions.dll b/src/lib/Azure.Core/NetCoreAssemblies/System.Threading.Tasks.Extensions.dll new file mode 100644 index 0000000000..ff691490b4 Binary files /dev/null and b/src/lib/Azure.Core/NetCoreAssemblies/System.Threading.Tasks.Extensions.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/Azure.Core.dll b/src/lib/Azure.Core/PreloadAssemblies/Azure.Core.dll new file mode 100644 index 0000000000..e52829cd90 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/Azure.Core.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/Microsoft.Bcl.AsyncInterfaces.dll b/src/lib/Azure.Core/PreloadAssemblies/Microsoft.Bcl.AsyncInterfaces.dll new file mode 100644 index 0000000000..0a784888ba Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/Microsoft.Bcl.AsyncInterfaces.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.4.0.4.0.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.4.0.4.0.dll new file mode 100644 index 0000000000..a2b54fb042 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.4.0.4.0.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.dll new file mode 100644 index 0000000000..917a04adfb Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Diagnostics.DiagnosticSource.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Memory.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Memory.dll new file mode 100644 index 0000000000..bdfc501e96 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Memory.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Numerics.Vectors.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Numerics.Vectors.dll new file mode 100644 index 0000000000..a808165acc Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Numerics.Vectors.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.0.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.0.dll new file mode 100644 index 0000000000..0b45903440 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.0.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.1.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.1.dll new file mode 100644 index 0000000000..3156239266 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.4.1.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.5.0.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.5.0.dll new file mode 100644 index 0000000000..0c27a0e21c Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Runtime.CompilerServices.Unsafe.4.0.5.0.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Security.AccessControl.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Security.AccessControl.dll new file mode 100644 index 0000000000..e8074324cd Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Security.AccessControl.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Security.Permissions.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Security.Permissions.dll new file mode 100644 index 0000000000..d1af38f0f8 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Security.Permissions.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Security.Principal.Windows.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Security.Principal.Windows.dll new file mode 100644 index 0000000000..afd187c149 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Security.Principal.Windows.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Text.Encodings.Web.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Text.Encodings.Web.dll new file mode 100644 index 0000000000..f31e26c725 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Text.Encodings.Web.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Text.Json.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Text.Json.dll new file mode 100644 index 0000000000..0491dede3b Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Text.Json.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.2.0.0.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.2.0.0.dll new file mode 100644 index 0000000000..d98daeaa09 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.2.0.0.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.3.1.0.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.3.1.0.dll new file mode 100644 index 0000000000..ff691490b4 Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Threading.Tasks.Extensions.4.3.1.0.dll differ diff --git a/src/lib/Azure.Core/PreloadAssemblies/System.Xml.ReaderWriter.dll b/src/lib/Azure.Core/PreloadAssemblies/System.Xml.ReaderWriter.dll new file mode 100644 index 0000000000..022e63a21a Binary files /dev/null and b/src/lib/Azure.Core/PreloadAssemblies/System.Xml.ReaderWriter.dll differ diff --git a/src/lib/Microsoft.Azure.Storage.DataMovement.dll b/src/lib/Microsoft.Azure.Storage.DataMovement.dll index bf5b4e3539..c1822d7986 100644 Binary files a/src/lib/Microsoft.Azure.Storage.DataMovement.dll and b/src/lib/Microsoft.Azure.Storage.DataMovement.dll differ diff --git a/tools/AzureRM.Example.psm1 b/tools/AzureRM.Example.psm1 index 8b65bb04e6..978cc858e6 100644 --- a/tools/AzureRM.Example.psm1 +++ b/tools/AzureRM.Example.psm1 @@ -55,7 +55,13 @@ if($PSEdition -eq 'Desktop' -and (Test-Path $preloadPath -ErrorAction Ignore)) try { Get-ChildItem -ErrorAction Stop -Path $preloadPath -Filter "*.dll" | ForEach-Object { - Add-Type -Path $_.FullName -ErrorAction Ignore | Out-Null + try + { + Add-Type -Path $_.FullName -ErrorAction Ignore | Out-Null + } + catch { + Write-Warning $_ + } } } catch {} @@ -72,7 +78,13 @@ if($PSEdition -eq 'Core' -and (Test-Path $netCorePath -ErrorAction Ignore)) $matches = ($loadedAssemblies | Where-Object {$_.Name -eq $assemblyName.Name}) if (-not $matches) { - Add-Type -Path $_.FullName -ErrorAction Ignore | Out-Null + try + { + Add-Type -Path $_.FullName -ErrorAction Ignore | Out-Null + } + catch { + Write-Warning $_ + } } } } diff --git a/tools/CheckAssemblies.ps1 b/tools/CheckAssemblies.ps1 index c584096f6d..3e67ad602e 100644 --- a/tools/CheckAssemblies.ps1 +++ b/tools/CheckAssemblies.ps1 @@ -11,6 +11,22 @@ # limitations under the License. # ---------------------------------------------------------------------------------- + +function Get-PreloadAssemblies{ + param( + [Parameter(Mandatory=$True)] + [string] $ModuleFolder + ) + + $preloadAssemblies = @() + $preloadFolderName = $PSEdition -eq 'Core' ? "NetCoreAssemblies" : "PreloadAssemblies" + $preloadFolder = [System.IO.Path]::Combine($ModuleFolder, $preloadFolderName) + if(Test-Path $preloadFolder){ + $preloadAssemblies = (Get-ChildItem $preloadFolder -Filter "*.dll").Name | ForEach-Object { $_ -replace ".dll", ""} + } + $preloadAssemblies +} + param( [ValidateNotNullOrEmpty()] [ValidateSet('Debug', 'Release')] @@ -38,6 +54,7 @@ foreach ($ModuleManifest in $ModuleManifestFiles) { $LoadedAssemblies += $ModuleMetadata.RequiredAssemblies } + $LoadedAssemblies += Get-PreloadAssemblies $ModuleManifest.Directory $LoadedAssemblies += $ModuleMetadata.NestedModules if ($ModuleMetadata.RequiredModules) { @@ -58,6 +75,7 @@ foreach ($ModuleManifest in $ModuleManifestFiles) { } $LoadedAssemblies += $ModuleMetadata.NestedModules } + $LoadedAssemblies += Get-PreloadAssemblies $RequiredModuleManifest.Directory } } diff --git a/tools/StaticAnalysis/DependencyAnalyzer/DependencyAnalyzer.cs b/tools/StaticAnalysis/DependencyAnalyzer/DependencyAnalyzer.cs index 883bc7168d..d463a2696c 100644 --- a/tools/StaticAnalysis/DependencyAnalyzer/DependencyAnalyzer.cs +++ b/tools/StaticAnalysis/DependencyAnalyzer/DependencyAnalyzer.cs @@ -384,7 +384,7 @@ namespace StaticAnalysis.DependencyAnalyzer foreach (var file in Directory.GetFiles(directoryPath).Where(file => file.EndsWith(".dll"))) { var assembly = CreateAssemblyRecord(file); - if (!IsFrameworkAssembly(assembly.Name)) + if (assembly?.Name != null && !IsFrameworkAssembly(assembly.Name)) { _assemblies[assembly.Name] = assembly; AddSharedAssembly(assembly); diff --git a/tools/StaticAnalysis/Exceptions/Az.Storage/BreakingChangeIssues.csv b/tools/StaticAnalysis/Exceptions/Az.Storage/BreakingChangeIssues.csv index 43aae78fb8..bd9325d856 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Storage/BreakingChangeIssues.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Storage/BreakingChangeIssues.csv @@ -159,4 +159,6 @@ "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.NewAzureStorageShareCommand","New-AzRmStorageShare","0","3000","The type of property 'Sku' of type 'Microsoft.Azure.Commands.Management.Storage.Models.PSStorageAccount' has changed from 'Microsoft.Azure.Management.Storage.Models.Sku' to 'Microsoft.Azure.Commands.Management.Storage.Models.PSSku'.","Change the type of property 'Sku' back to 'Microsoft.Azure.Management.Storage.Models.Sku'." "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.RemoveAzureStorageShareCommand","Remove-AzRmStorageShare","0","3000","The type of property 'Sku' of type 'Microsoft.Azure.Commands.Management.Storage.Models.PSStorageAccount' has changed from 'Microsoft.Azure.Management.Storage.Models.Sku' to 'Microsoft.Azure.Commands.Management.Storage.Models.PSSku'.","Change the type of property 'Sku' back to 'Microsoft.Azure.Management.Storage.Models.Sku'." "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.UpdateAzureStorageShareCommand","Update-AzRmStorageShare","0","3000","The type of property 'Sku' of type 'Microsoft.Azure.Commands.Management.Storage.Models.PSStorageAccount' has changed from 'Microsoft.Azure.Management.Storage.Models.Sku' to 'Microsoft.Azure.Commands.Management.Storage.Models.PSSku'.","Change the type of property 'Sku' back to 'Microsoft.Azure.Management.Storage.Models.Sku'." - "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.SetAzureStorageContainerImmutabilityPolicyCommand","Set-AzRmStorageContainerImmutabilityPolicy","0","1050","The parameter set '__AllParameterSets' for cmdlet 'Set-AzRmStorageContainerImmutabilityPolicy' has been removed.","Add parameter set '__AllParameterSets' back to cmdlet 'Set-AzRmStorageContainerImmutabilityPolicy'." \ No newline at end of file + "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.SetAzureStorageContainerImmutabilityPolicyCommand","Set-AzRmStorageContainerImmutabilityPolicy","0","1050","The parameter set '__AllParameterSets' for cmdlet 'Set-AzRmStorageContainerImmutabilityPolicy' has been removed.","Add parameter set '__AllParameterSets' back to cmdlet 'Set-AzRmStorageContainerImmutabilityPolicy'." +"Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.InvokeAzureStorageAccountFailoverCommand","Invoke-AzStorageAccountFailover","0","3000","The type of property 'Sku' of type 'Microsoft.Azure.Commands.Management.Storage.Models.PSStorageAccount' has changed from 'Microsoft.Azure.Management.Storage.Models.Sku' to 'Microsoft.Azure.Commands.Management.Storage.Models.PSSku'.","Change the type of property 'Sku' back to 'Microsoft.Azure.Management.Storage.Models.Sku'." +"Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.RevokeAzureStorageAccountUserDelegationKeysCommand","Revoke-AzStorageAccountUserDelegationKeys","0","3000","The type of property 'Sku' of type 'Microsoft.Azure.Commands.Management.Storage.Models.PSStorageAccount' has changed from 'Microsoft.Azure.Management.Storage.Models.Sku' to 'Microsoft.Azure.Commands.Management.Storage.Models.PSSku'.","Change the type of property 'Sku' back to 'Microsoft.Azure.Management.Storage.Models.Sku'." \ No newline at end of file diff --git a/tools/StaticAnalysis/Exceptions/Az.Storage/MissingAssemblies.csv b/tools/StaticAnalysis/Exceptions/Az.Storage/MissingAssemblies.csv index 4926fc3d8c..f3c184037a 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Storage/MissingAssemblies.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Storage/MissingAssemblies.csv @@ -2,4 +2,7 @@ "Az.Storage","Mono.Posix.NETStandard","1.0.0.0","Microsoft.WindowsAzure.Storage.DataMovement","0","3000","Missing assembly Mono.Posix.NETStandard referenced from Microsoft.WindowsAzure.Storage.DataMovement","Ensure that the assembly is included in the Wix file or directory" "Az.Storage","Microsoft.Win32.Registry","4.1.1.0","System.Management.Automation","0","3000","Missing assembly Microsoft.Win32.Registry referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" "Az.Storage","Microsoft.PowerShell.CoreCLR.Eventing","6.1.0.0","System.Management.Automation","0","3000","Missing assembly Microsoft.PowerShell.CoreCLR.Eventing referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" -"Az.Storage","Mono.Posix.NETStandard","1.0.0.0","Microsoft.Azure.Storage.DataMovement","0","3000","Missing assembly Mono.Posix.NETStandard referenced from Microsoft.Azure.Storage.DataMovement","Ensure that the assembly is included in the Wix file or directory" \ No newline at end of file +"Az.Storage","Mono.Posix.NETStandard","1.0.0.0","Microsoft.Azure.Storage.DataMovement","0","3000","Missing assembly Mono.Posix.NETStandard referenced from Microsoft.Azure.Storage.DataMovement","Ensure that the assembly is included in the Wix file or directory" +"Az.Storage","System.Text.Encodings.Web","4.0.4.0","System.Text.Json","0","3000","Missing assembly System.Text.Encodings.Web referenced from System.Text.Json","Ensure that the assembly is included in the Wix file or directory" +"Az.Storage","System.Numerics.Vectors","4.1.3.0","System.Text.Json","0","3000","Missing assembly System.Numerics.Vectors referenced from System.Text.Json","Ensure that the assembly is included in the Wix file or directory" +"Az.Storage","System.Runtime.CompilerServices.Unsafe","4.0.5.0","System.Text.Json","0","3000","Missing assembly System.Runtime.CompilerServices.Unsafe referenced from System.Text.Json","Ensure that the assembly is included in the Wix file or directory" \ No newline at end of file diff --git a/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv b/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv index 043165bcba..023ff218ff 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Storage/SignatureIssues.csv @@ -61,3 +61,4 @@ "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.NewAzureStorageAccountManagementPolicyRuleCommand","New-AzStorageAccountManagementPolicyRule","1","8100","New-AzStorageAccountManagementPolicyRule Does not support ShouldProcess but the cmdlet verb New indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.EnableAzStorageBlobDeleteRetentionPolicyCommand","Enable-AzStorageBlobDeleteRetentionPolicy","1","8410","Parameter RetentionDays of cmdlet Enable-AzStorageBlobDeleteRetentionPolicy does not follow the enforced naming convention of using a singular noun for a parameter name.","Consider using a singular noun for the parameter name." "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management.dll","Microsoft.Azure.Commands.Management.Storage.RevokeAzureStorageAccountUserDelegationKeysCommand","Revoke-AzStorageAccountUserDelegationKeys","1","8400","Revoke-AzStorageAccountUserDelegationKeys uses the noun 'AzStorageAccountUserDelegationKeys', which does not follow the enforced naming convention of using a singular noun for a cmdlet name.","Consider using a singular noun for the cmdlet name." +"Microsoft.Azure.PowerShell.Cmdlets.Storage.dll","Microsoft.Azure.Commands.Management.Storage.NewAzDataLakeGen2ItemAclObjectCommand","New-AzDataLakeGen2ItemAclObject","1","8100","New-AzDataLakeGen2ItemAclObject Does not support ShouldProcess but the cmdlet verb New indicates that it should.","Determine if the cmdlet should implement ShouldProcess and if so determine if it should implement Force / ShouldContinue" \ No newline at end of file