support for runtime packages on NuGet (#3281)

bugfix when exactly one matching package on Azure DevOps

Example of runtime packages here
https://dev.azure.com/freddydk/apps/_artifacts/feed/MyRuntimePackages

Co-authored-by: freddydk <freddydk@users.noreply.github.com>

Common/Download-File.ps1

@@ -7,6 +7,8 @@
   Url from which the file will get downloaded
  .Parameter destinationFile
   Destinatin for the downloaded file
+ .Parameter description
+  Description for the download process
  .Parameter Headers
   Specify a custom header for the request
  .Parameter dontOverwrite
@@ -22,6 +24,7 @@ function Download-File {
         [string] $sourceUrl,
         [Parameter(Mandatory=$true)]
         [string] $destinationFile,
+        [string] $description = '',
         [hashtable] $headers = @{"UserAgent" = "BcContainerHelper $bcContainerHelperVersion" },
         [switch] $dontOverwrite,
         [int]    $timeout = 100
@@ -73,7 +76,12 @@ try {
         New-Item -Path $path -ItemType Directory -Force | Out-Null
     }
     [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
-    Write-Host "Downloading $destinationFile"
+    if ($description) {
+        Write-Host "Downloading $description to $destinationFile"
+    }
+    else {
+        Write-Host "Downloading $destinationFile"
+    }
     if ($sourceUrl -like "https://*.sharepoint.com/*download=1*") {
         Invoke-WebRequest -UseBasicParsing -Uri $sourceUrl -OutFile $destinationFile
     }

ContainerHandling/Get-LatestAlLanguageExtensionUrl.ps1

@@ -17,7 +17,8 @@ function Get-LatestAlLanguageExtensionUrl {
 
 $telemetryScope = InitTelemetryScope -name $MyInvocation.InvocationName -parameterValues $PSBoundParameters -includeParameters @()
 try {
-    GetLatestAlLanguageExtensionUrl -allowPrerelease:$allowPrerelease
+    $version, $url = GetLatestAlLanguageExtensionVersionAndUrl -allowPrerelease:$allowPrerelease
+    return $url
 }
 catch {
     TrackException -telemetryScope $telemetryScope -errorRecord $_

HelperFunctions.ps1

@@ -534,7 +534,7 @@ function CopyAppFilesToFolder {
     if (!(Test-Path $folder)) {
         New-Item -Path $folder -ItemType Directory | Out-Null
     }
-    $appFiles | Where-Object { $_ } | % {
+    $appFiles | Where-Object { $_ } | ForEach-Object {
         $appFile = $_
         if ($appFile -like "http://*" -or $appFile -like "https://*") {
             $appUrl = $appFile
@@ -567,7 +567,7 @@ function CopyAppFilesToFolder {
                             $copied = $true
                         }
                         Expand-Archive $appfile -DestinationPath $tmpFolder -Force
-                        Get-ChildItem -Path $tmpFolder -Recurse | Where-Object { $_.Name -like "*.app" -or $_.Name -like "*.zip" } | % {
+                        Get-ChildItem -Path $tmpFolder -Recurse | Where-Object { $_.Name -like "*.app" -or $_.Name -like "*.zip" } | ForEach-Object {
                             CopyAppFilesToFolder -appFile $_.FullName -folder $folder
                         }
                     }
@@ -1228,7 +1228,7 @@ function GetAppInfo {
     Write-Host "::endgroup::"
 }
 
-function GetLatestAlLanguageExtensionUrl {
+function GetLatestAlLanguageExtensionVersionAndUrl {
     Param(
         [switch] $allowPrerelease
     )
@@ -1238,19 +1238,18 @@ function GetLatestAlLanguageExtensionUrl {
                       -Body '{"filters":[{"criteria":[{"filterType":8,"value":"Microsoft.VisualStudio.Code"},{"filterType":12,"value":"4096"},{"filterType":7,"value":"ms-dynamics-smb.al"}],"pageNumber":1,"pageSize":50,"sortBy":0,"sortOrder":0}],"assetTypes":[],"flags":0x192}' `
                       -ContentType application/json | ConvertFrom-Json
     
-    $vsixUrl =  $listing.results | Select-Object -First 1 -ExpandProperty extensions `
+    $result =  $listing.results | Select-Object -First 1 -ExpandProperty extensions `
                          | Select-Object -ExpandProperty versions `
                          | Where-Object { ($allowPrerelease.IsPresent -or !(($_.properties.Key -eq 'Microsoft.VisualStudio.Code.PreRelease') -and ($_.properties | where-object { $_.Key -eq 'Microsoft.VisualStudio.Code.PreRelease' }).value -eq "true")) } `
-                         | Select-Object -First 1 -ExpandProperty files `
-                         | Where-Object { $_.assetType -eq "Microsoft.VisualStudio.Services.VSIXPackage"} `
-                         | Select-Object -ExpandProperty source
-     
-    if ($vsixUrl) {
-        $vsixUrl
-    }
-    else {
-        throw "Unable to locate latest AL Language Extension from the VS Code Marketplace"
+                         | Select-Object -First 1
+
+    if ($result) {
+        $vsixUrl = $result.files | Where-Object { $_.assetType -eq "Microsoft.VisualStudio.Services.VSIXPackage"} | Select-Object -ExpandProperty source
+        if ($vsixUrl) {
+            return $result.version, $vsixUrl
+        }
     }
+    throw "Unable to locate latest AL Language Extension from the VS Code Marketplace"
 }
 
 function DetermineVsixFile {
@@ -1261,13 +1260,84 @@ function DetermineVsixFile {
     if ($vsixFile -eq 'default') {
         return ''
     }
-    elseif ($vsixFile -eq 'latest') {
-        return (GetLatestAlLanguageExtensionUrl)
-    }
-    elseif ($vsixFile -eq 'preview') {
-        return (GetLatestAlLanguageExtensionUrl -allowPrerelease)
+    elseif ($vsixFile -eq 'latest' -or $vsixFile -eq 'preview') {
+        $version, $url = GetLatestAlLanguageExtensionVersionAndUrl -allowPrerelease:($vsixFile -eq 'preview')
+        return $url
     }
     else {
         return $vsixFile
     }
 }
+
+# Cached Path for latest and preview versions of AL Language Extension
+$AlLanguageExtenssionPath = @('','')
+
+function DownloadLatestAlLanguageExtension {
+    Param(
+        [switch] $allowPrerelease
+    )
+
+    $mutexName = "DownloadAlLanguageExtension"
+    $mutex = New-Object System.Threading.Mutex($false, $mutexName)
+    try {
+        try {
+            if (!$mutex.WaitOne(1000)) {
+                Write-Host "Waiting for other process downloading AL Language Extension"
+                $mutex.WaitOne() | Out-Null
+                Write-Host "Other process completed downloading"
+            }
+        }
+        catch [System.Threading.AbandonedMutexException] {
+           Write-Host "Other process terminated abnormally"
+        }
+
+        # Check if we already have the latest version downloaded and located in this session
+        if ($script:AlLanguageExtenssionPath[$allowPrerelease.IsPresent]) {
+            $path = $script:AlLanguageExtenssionPath[$allowPrerelease.IsPresent]
+            if (Test-Path $path -PathType Container) {
+                return $path
+            }
+            else {
+                $script:AlLanguageExtenssionPath[$allowPrerelease.IsPresent] = ''
+            }
+        }
+
+        $version, $url = GetLatestAlLanguageExtensionVersionAndUrl -allowPrerelease:$allowPrerelease
+        $path = Join-Path $bcContainerHelperConfig.hostHelperFolder "alLanguageExtension/$version"
+        if (!(Test-Path $path -PathType Container)) {
+            $AlLanguageExtensionsFolder = Join-Path $bcContainerHelperConfig.hostHelperFolder "alLanguageExtension"
+            if (!(Test-Path $AlLanguageExtensionsFolder -PathType Container)) {
+                New-Item -Path $AlLanguageExtensionsFolder -ItemType Directory | Out-Null
+            }
+            $description = "AL Language Extension"
+            if ($allowPrerelease) {
+                $description += " (Prerelease)"
+            }
+            $zipFile = "$path.zip"
+            Download-File -sourceUrl $url -destinationFile $zipFile -Description $description
+            Expand-7zipArchive -Path $zipFile -DestinationPath $path
+            Remove-Item -Path $zipFile -Force
+        }
+        $script:AlLanguageExtenssionPath[$allowPrerelease.IsPresent] = $path
+        return $path
+    }
+    finally {
+        $mutex.ReleaseMutex()
+    }
+}
+
+function GetAppJsonFromAppFile {
+    Param(
+        [string] $appFile
+    )
+    # ALTOOL is at the moment only available in prerelease        
+    $path = DownloadLatestAlLanguageExtension -allowPrerelease
+    if ($isWindows) {
+        $alToolExe = Join-Path $path 'extension/bin/win32/altool.exe'
+    }
+    else {
+        $alToolExe = Join-Path $path 'extension/bin/linux/altool'
+    }
+    $appJson = CmdDo -Command $alToolExe -arguments @('GetPackageManifest', """$appFile""") -returnValue -silent | ConvertFrom-Json
+    return $appJson
+}

NuGet/Download-BcNuGetPackageToFolder.ps1

@@ -169,7 +169,7 @@ Function Download-BcNuGetPackageToFolder {
                         throw $dependenciesErr
                     }
                     else {
-                        # If we are looking for the latest version, then we can try to find another version
+                        # If we are looking for the latest matching version, then we can try to find another version
                         Write-Host "WARNING: $dependenciesErr"
                         break
                     }
@@ -183,8 +183,9 @@ Function Download-BcNuGetPackageToFolder {
                 }
             }
             if ($dependenciesErr) {
-                # If we are looking for the latest version, then we can try to find another version
+                # If we are looking for the latest matching version, then we can try to find another version
                 $excludeVersions += $packageVersion
+                Remove-Item -Path $package -Recurse -Force
                 continue
             }
             $appFiles = (Get-Item -Path (Join-Path $package '*.app')).FullName

NuGet/New-BcNuGetPackage.ps1

@@ -58,7 +58,12 @@ Function New-BcNuGetPackage {
         [Parameter(Mandatory=$false)]
         [string] $applicationDependencyId = 'Microsoft.Application',
         [Parameter(Mandatory=$false)]
+        [string] $applicationDependency = '',
+        [Parameter(Mandatory=$false)]
         [string] $platformDependencyId = 'Microsoft.Platform',
+        [Parameter(Mandatory=$false)]
+        [string] $runtimeDependencyId = '{publisher}.{name}.runtime-{version}',
+        [switch] $isIndirectPackage,
         [obsolete('NuGet Dependencies are always included.')]
         [switch] $includeNuGetDependencies
     )
@@ -75,15 +80,14 @@ Function New-BcNuGetPackage {
         throw "Unable to locate file: $_"
     }
     $appFile = (Get-Item $appfile).FullName
-    $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString())
     $rootFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString())
     New-Item -Path $rootFolder -ItemType Directory | Out-Null
     try {
-        Copy-Item -Path $appFile -Destination $rootFolder -Force
-        Extract-AppFileToFolder -appFilename $appFile -generateAppJson -appFolder $tmpFolder
-        $appJsonFile = Join-Path $tmpFolder 'app.json'
-        $appJson = Get-Content $appJsonFile -Encoding UTF8 | ConvertFrom-Json
-        $packageId = $packageId.replace('{id}',$appJson.id).replace('{name}',[nuGetFeed]::Normalize($appJson.name)).replace('{publisher}',[nuGetFeed]::Normalize($appJson.publisher))
+        if (!$isIndirectPackage.IsPresent) {
+            Copy-Item -Path $appFile -Destination $rootFolder -Force
+        }
+        $appJson = GetAppJsonFromAppFile -appFile $appFile
+        $packageId = $packageId.replace('{id}',$appJson.id).replace('{name}',[nuGetFeed]::Normalize($appJson.name)).replace('{publisher}',[nuGetFeed]::Normalize($appJson.publisher)).replace('{version}',$appJson.version.replace('.','-'))
         if ($null -eq $packageVersion) {
             $packageVersion = [System.Version]$appJson.version
         }
@@ -99,6 +103,11 @@ Function New-BcNuGetPackage {
         if (-not $packageAuthors) {
             $packageAuthors = $appJson.publisher
         }
+        if (-not $applicationDependency) {
+            if ($appJson.PSObject.Properties.Name -eq 'Application' -and $appJson.Application) {
+                $applicationDependency = $appJson.Application
+            }
+        }
 
         if ($prereleaseTag) {
             $packageVersionStr = "$($packageVersion)-$prereleaseTag"
@@ -132,17 +141,19 @@ Function New-BcNuGetPackage {
             $XmlObjectWriter.WriteEndElement()
         }
         $XmlObjectWriter.WriteStartElement("dependencies")
-        $appJson.dependencies | ForEach-Object {
-            $id = $dependencyIdTemplate.replace('{id}',$_.id).replace('{name}',[nuGetFeed]::Normalize($_.name)).replace('{publisher}',[nuGetFeed]::Normalize($_.publisher))
-            $XmlObjectWriter.WriteStartElement("dependency")
-            $XmlObjectWriter.WriteAttributeString("id", $id)
-            $XmlObjectWriter.WriteAttributeString("version", $_.Version)
-            $XmlObjectWriter.WriteEndElement()
+        if ($appJson.PSObject.Properties.Name -eq 'dependencies') {
+            $appJson.dependencies | ForEach-Object {
+                $id = $dependencyIdTemplate.replace('{id}',$_.id).replace('{name}',[nuGetFeed]::Normalize($_.name)).replace('{publisher}',[nuGetFeed]::Normalize($_.publisher))
+                $XmlObjectWriter.WriteStartElement("dependency")
+                $XmlObjectWriter.WriteAttributeString("id", $id)
+                $XmlObjectWriter.WriteAttributeString("version", $_.Version)
+                $XmlObjectWriter.WriteEndElement()
+            }
         }
-        if ($appJson.PSObject.Properties.Name -eq 'Application' -and $appJson.Application) {
+        if ($applicationDependency) {
             $XmlObjectWriter.WriteStartElement("dependency")
             $XmlObjectWriter.WriteAttributeString("id", $applicationDependencyId)
-            $XmlObjectWriter.WriteAttributeString("version", $appJson.Application)
+            $XmlObjectWriter.WriteAttributeString("version", $applicationDependency)
             $XmlObjectWriter.WriteEndElement()
         }
         if ($appJson.PSObject.Properties.Name -eq 'Platform' -and  $appJson.Platform) {
@@ -151,15 +162,24 @@ Function New-BcNuGetPackage {
             $XmlObjectWriter.WriteAttributeString("version", $appJson.Platform)
             $XmlObjectWriter.WriteEndElement()
         }
+        if ($isIndirectPackage.IsPresent) {
+            $XmlObjectWriter.WriteStartElement("dependency")
+            $id = $runtimeDependencyId.replace('{id}',$appJson.id).replace('{name}',[nuGetFeed]::Normalize($appJson.name)).replace('{publisher}',[nuGetFeed]::Normalize($appJson.publisher)).replace('{version}',$appJson.version.replace('.','-'))
+            $XmlObjectWriter.WriteAttributeString("id", $id)
+            $XmlObjectWriter.WriteAttributeString("version", '1.0.0.0')
+            $XmlObjectWriter.WriteEndElement()
+        }
         $XmlObjectWriter.WriteEndElement()
         $XmlObjectWriter.WriteEndElement()
-        $XmlObjectWriter.WriteStartElement("files")
-        $XmlObjectWriter.WriteStartElement("file")
-        $appFileName = [System.IO.Path]::GetFileName($appfile)
-        $XmlObjectWriter.WriteAttributeString("src", $appFileName );
-        $XmlObjectWriter.WriteAttributeString("target", $appFileName);
-        $XmlObjectWriter.WriteEndElement()
-        $XmlObjectWriter.WriteEndElement()
+        if (!$isIndirectPackage.IsPresent) {
+            $XmlObjectWriter.WriteStartElement("files")
+            $XmlObjectWriter.WriteStartElement("file")
+            $appFileName = [System.IO.Path]::GetFileName($appfile)
+            $XmlObjectWriter.WriteAttributeString("src", $appFileName );
+            $XmlObjectWriter.WriteAttributeString("target", $appFileName);
+            $XmlObjectWriter.WriteEndElement()
+            $XmlObjectWriter.WriteEndElement()
+        }
         $XmlObjectWriter.WriteEndElement()
         $XmlObjectWriter.WriteEndDocument()
         $XmlObjectWriter.Flush()
@@ -189,7 +209,6 @@ Function New-BcNuGetPackage {
         $nupkgFile
     }
     finally {
-        Remove-Item -Path $tmpFolder -Recurse -Force -ErrorAction SilentlyContinue
         Remove-Item -Path $rootFolder -Recurse -Force -ErrorAction SilentlyContinue
     }
 }

NuGet/NuGetFeedClass.ps1

@@ -103,7 +103,7 @@ class NuGetFeed {
                 if ($result.Count -eq 0) {
                     break
                 }
-                $matching += @($result | Where-Object { $_.name -like "*$packageName*" -and $this.IsTrusted($_.name) }) | ForEach-Object { $_.name }
+                $matching += @($result | Where-Object { $_.name -like "*$packageName*" -and $this.IsTrusted($_.name) } | ForEach-Object { $_.name })
                 $page++
             }
         }
@@ -119,7 +119,7 @@ class NuGetFeed {
                 throw (GetExtendedErrorMessage $_)
             }
             # Check that the found pattern matches the package name and the trusted patterns
-            $matching = @($searchResult.data | Where-Object { $_.id -like "*$($packageName)*" -and $this.IsTrusted($_.id) }) | ForEach-Object { $_.id }
+            $matching = @($searchResult.data | Where-Object { $_.id -like "*$($packageName)*" -and $this.IsTrusted($_.id) } | ForEach-Object { $_.id })
         }
         Write-Host "$($matching.count) matching packages found"
         return $matching | ForEach-Object { Write-Host "- $_"; $_ }
@@ -185,7 +185,8 @@ class NuGetFeed {
             else {
                 $fromver = [System.Version]::new(0,0,0,0)
                 if ($inclFrom) {
-                    throw "Invalid NuGet version range $nuGetVersionRange"
+                    Write-Host "Invalid NuGet version range $nuGetVersionRange"
+                    return $false
                 }
             }
             if ($matches[4]) {
@@ -194,14 +195,16 @@ class NuGetFeed {
             elseif ($range) {
                 $tover = [System.Version]::new([int32]::MaxValue,[int32]::MaxValue,[int32]::MaxValue,[int32]::MaxValue)
                 if ($inclTo) {
-                    throw "Invalid NuGet version range $nuGetVersionRange"
+                    Write-Host "Invalid NuGet version range $nuGetVersionRange"
+                    return $false
                 }
             }
             else {
                 $tover = $fromver
             }
             if (!$range -and (!$inclFrom -or !$inclTo)) {
-                throw "Invalid NuGet version range $nuGetVersionRange"
+                Write-Host "Invalid NuGet version range $nuGetVersionRange"
+                return $false
             }
             if ($inclFrom) {
                 if ($inclTo) {