From 3599b1f1bcf2a4c42c711718353f8e8a5c5bf99f Mon Sep 17 00:00:00 2001 From: BruceHaley Date: Tue, 25 Jan 2022 10:33:32 -0800 Subject: [PATCH] Create pipelines to unlist deprecated packages on 4 feeds (#5827) * Add UnlistPackagesOnMyGet.ps1 * Fix $packageNames arg * Add debug trace * Remove debug trace * Fix $packageNames arg * Fix if ($deletePackagesForReal * Fix [boolean]$deletePackagesForReal * Fix ($deletePackagesForReal -eq true) * Fix $deletePackagesForReal -eq "true" * Fix $deletePackagesForReal = "false" * Log "deletePackagesForReal: " * Tweaks * Enable real delete * Tweak comment * Add UnlistPackagesOnNuGet.ps1 * Tweak string array declaration * Tweak descriptions * Tweak 2 on description * Fix versions getting * Tweak arg, comment * Set $versionToUnlist * Fix unlistPackagesForReal arg and msg * Fix $RegistryUrlSource * Tweak logging. * Tweak logging 2 * Add UnlistNpmPackagesOnMyGet.ps1 * Tweaks 3 * Add debug lines * More debug lines * More debug 3 * Strip debug lines * Set up npm unpublish command * Remove --registry=$feedApiUrl args from unpublish * Trying nuget delete again. * Trying the $feedApiUrl * Try npm config set registry * Prepend "v" to version number for npm unpublish * Add UnlistPackagesOnNpm.ps1 * Add --loglevel verbose * Add --loglevel verbose 2 * Try @microsoft/ * Try for real unpublish verbose * Switch arg to $npmPersonalAccessToken * Comment * Try unscoped again. * Add $unlistAllOlderPackagesAlso flag * Fix "if" * Reformatted output. * Add $unlistOlderVersionsAlso flag * Add $unlistOlderVersionsAlso flag to azure script * Add $unlistOlderVersionsAlso flag to npm pipeline * Tweak * Add $unlistOlderVersionsAlso flag to npm on myget * Add $unlistOlderVersionsAlso flag to NuGet pipelins * Fix single version to unlist in NuGet * Fix $versionToUnlist matching * Revise matching of $versionToUnlist 2 * Try resetting $index * Add debugging * Try again with multiple qualifying versions * Fixed now. * Cleanup. * Cleanup 2 * Fix the other pipelines to match * Fix npm on MyGet * Tweak npm on myget --- build/UnlistNpmPackagesOnMyGet.ps1 | 95 ++++++++++++++++++++++++++++++ build/UnlistPackagesOnAzure.ps1 | 74 +++++++++++++++++++++++ build/UnlistPackagesOnMyGet.ps1 | 88 +++++++++++++++++++++++++++ build/UnlistPackagesOnNpm.ps1 | 74 +++++++++++++++++++++++ build/UnlistPackagesOnNuGet.ps1 | 76 ++++++++++++++++++++++++ 5 files changed, 407 insertions(+) create mode 100644 build/UnlistNpmPackagesOnMyGet.ps1 create mode 100644 build/UnlistPackagesOnAzure.ps1 create mode 100644 build/UnlistPackagesOnMyGet.ps1 create mode 100644 build/UnlistPackagesOnNpm.ps1 create mode 100644 build/UnlistPackagesOnNuGet.ps1 diff --git a/build/UnlistNpmPackagesOnMyGet.ps1 b/build/UnlistNpmPackagesOnMyGet.ps1 new file mode 100644 index 000000000..ba361d888 --- /dev/null +++ b/build/UnlistNpmPackagesOnMyGet.ps1 @@ -0,0 +1,95 @@ +# +# Shows but cannot unlist npm package versions on MyGet.org lower than or equal to $versionToUnlist. +# Cannot unlist because per MyGet support, "we do not support the npm unpublish command". Unlisting must be done in the UI. +# +param +( + [string]$versionToUnlist = "4.0.5-1500", + [string]$unlistOlderVersionsAlso = "false", + [string[]]$packageNames = @( "adaptive-expressions","botbuilder","botbuilder-dialogs-adaptive-runtime-core" ), + [string]$myGetFeedName = "botbuilder-v4-js-daily", + [string]$myGetPersonalAccessToken, + [string]$unlistPackagesForReal = "false" +) + +$feedStateUrl = "https://botbuilder.myget.org/F/$myGetFeedName/auth/$myGetPersonalAccessToken/api/v2/feed-state"; +#$feedApiUrl = "https://botbuilder.myget.org/F/$myGetFeedName/npm/"; +$feedApiUrl = "https://botbuilder.myget.org/F/$myGetFeedName/api/v3/index.json"; + +Function Sort-Versions +{ + param ( [string]$versions ); + + New-Item -Path .\xxx.csv -ItemType "file" -Value $versions -Force | Out-Null; + + $Header = 'Major', 'Minor', 'Build', 'Revision', 'p5', 'p6'; + + $P = Import-Csv -Path .\xxx.csv -Delimiter . -Header $Header; + $P | % { $_.Major = [int]$_.Major }; + $P | % { $_.Minor = [int]$_.Minor }; + $P = $P | Sort -Property Major,Minor,Build,Revision; + #$P | Format-Table; + + $Q = $P | % { ($_.PSObject.Properties | % { $_.Value }) -join '.'}; + + return $Q.TrimEnd("."); +} + +"versionToUnlist: " + $versionToUnlist; +"unlistOlderVersionsAlso: " + $unlistOlderVersionsAlso; +"unlistPackagesForReal: " + $unlistPackagesForReal; +" "; +"Package versions to unlist:" + +$result = Invoke-RestMethod -Uri $feedStateUrl -Method Get -ContentType "application/json"; + +npm config set registry https://botbuilder.myget.org/F/botbuilder-v4-js-daily/npm/; + +foreach ($packageName in $packageNames) { + $versionsToUnlist = $Null; + $index = -1; + + $packageInfo = $result.packages | Where-Object {$_.id -eq $packageName}; + + [string]$unsortedVersions = $packageInfo.versions | %{ $_ + "`r`n" }; + + $sortedVersions = Sort-Versions $unsortedVersions; + + if ($unlistOlderVersionsAlso -eq "true") { + for ([int]$i = 0; $i -lt $sortedVersions.Count; $i++) + { + if ($sortedVersions[$i] -ge $versionToUnlist) { + $index = $i; + if ($sortedVersions[$i] -gt $versionToUnlist) { $index--; } + break; + } + } + + if ($index -ne $Null -and $index -ge 0) { + [string[]]$versionsToUnlist = $sortedVersions | select -First ($index + 1); + } + } else { + [string[]]$versionsToUnlist = $sortedVersions.Where({$_ -eq $versionToUnlist}); + } + + if ($versionsToUnlist.Count -gt 0) { + "-----------------------------------------"; + $packageName + ":"; + "-----------------------------------------"; + + foreach ($version in $versionsToUnlist) { + #$url = "$feedApiUrl$packageName/versions/$version"; + if ($unlistPackagesForReal -eq "true") { + " Unlisting $version (nonfunctional"; + #" npm unpublish $packageName@v$version --loglevel verbose"; + #npm unpublish $packageName@v$version --loglevel verbose; + #Invoke-RestMethod -Uri $url -Method Delete -ContentType "application/json"; + #"nuget delete $packageName $version -Source $feedApiUrl -apikey $myGetPersonalAccessToken -NonInteractive"; + #nuget delete $packageName $version -Source $feedApiUrl -apikey $myGetPersonalAccessToken -NonInteractive; + } else { + " $version"; + } + } + } +} +"-----------------------------------------"; diff --git a/build/UnlistPackagesOnAzure.ps1 b/build/UnlistPackagesOnAzure.ps1 new file mode 100644 index 000000000..0228d4b61 --- /dev/null +++ b/build/UnlistPackagesOnAzure.ps1 @@ -0,0 +1,74 @@ +# +# Unlists NuGet package versions on Azure ConversationalAI feed lower than or equal to $versionToUnlist. +# Run this first with $unlistPackagesForReal = false (default) to verify what versions will be affected. +# See: https://stackoverflow.com/questions/34958908/where-can-i-find-documentation-for-the-nuget-v3-api +# +param +( + [string]$versionToUnlist = "4.11.2", + [string]$unlistOlderVersionsAlso = "false", + [string[]]$packageNames = @( "AdaptiveExpressions","Microsoft.Bot.Builder","Microsoft.Bot.Builder.Integration.AspNet.Core" ), + [string]$adoPersonalAccessToken, + [string]$unlistPackagesForReal = "false" +) + +$RegistryUrlSource = "https://pkgs.dev.azure.com/ConversationalAI/BotFramework/_packaging/SDK/nuget/v3/index.json"; + +Function Get-Versions-From-Azure +{ + param ( [string]$packageName ); + + $result = nuget list $packageName -Source "$RegistryUrlSource" -PreRelease -AllVersions | Where-Object { $_ -like "$packageName *" }; + $versions = $result | % { $_.Split(" ")[-1] }; + + return $versions +} + +"versionToUnlist: " + $versionToUnlist; +"unlistOlderVersionsAlso: " + $unlistOlderVersionsAlso; +"unlistPackagesForReal: " + $unlistPackagesForReal; +" "; +"Package versions to unlist:" + +foreach ($packageName in $packageNames) { + $versionsToUnlist = $Null; + $index = -1; + + $sortedVersions = Get-Versions-From-Azure $packageName; + + [array]::Reverse($sortedVersions); + + if ($unlistOlderVersionsAlso -eq "true") { + for ([int]$i = 0; $i -lt $sortedVersions.Count; $i++) + { + if ($sortedVersions[$i] -ge $versionToUnlist) { + $index = $i; + if ($sortedVersions[$i] -gt $versionToUnlist) { $index--; } + break; + } + } + + if ($index -ne $Null -and $index -ge 0) { + [string[]]$versionsToUnlist = $sortedVersions | select -First ($index + 1); + } + } else { + [string[]]$versionsToUnlist = $sortedVersions.Where({$_ -eq $versionToUnlist}); + } + + if ($versionsToUnlist.Count -gt 0) { + "-----------------------------------------"; + $packageName + ":"; + "-----------------------------------------"; + + foreach ($version in $versionsToUnlist) { + if ($unlistPackagesForReal -eq "true") { + " Unlisting $version"; + " nuget delete $packageName $version -Source $RegistryUrlSource -apikey $adoPersonalAccessToken -NonInteractive"; + nuget delete $packageName $version -Source $RegistryUrlSource -apikey $adoPersonalAccessToken -NonInteractive; + } else { + " $version"; + } + } + } +} +"-----------------------------------------"; diff --git a/build/UnlistPackagesOnMyGet.ps1 b/build/UnlistPackagesOnMyGet.ps1 new file mode 100644 index 000000000..8230bbaaf --- /dev/null +++ b/build/UnlistPackagesOnMyGet.ps1 @@ -0,0 +1,88 @@ +# +# Unlists NuGet packages on MyGet.org with the specified version number. Option to unlist all older versions as well. +# Run this first with $unlistPackagesForReal = false (default) to verify what versions will be affected. +# +param +( + [string]$versionToUnlist = "4.6.1-preview", + [string]$unlistOlderVersionsAlso = "true", + [string[]]$packageNames = @( "AdaptiveExpressions","Microsoft.Bot.Builder","Microsoft.Bot.Builder.Integration.AspNet.Core" ), + [string]$myGetFeedName = "botbuilder-v4-dotnet-daily", + [string]$myGetPersonalAccessToken, + [string]$unlistPackagesForReal = "false" +) + +$feedStateUrl = "https://botbuilder.myget.org/F/$myGetFeedName/auth/$myGetPersonalAccessToken/api/v2/feed-state"; +$feedApiUrl = "https://botbuilder.myget.org/F/$myGetFeedName/api/v3/index.json"; + +Function Sort-Versions +{ + param ( [string]$versions ); + + New-Item -Path .\xxx.csv -ItemType "file" -Value $versions -Force | Out-Null; + + $Header = 'Major', 'Minor', 'Build', 'Revision', 'p5', 'p6'; + + $P = Import-Csv -Path .\xxx.csv -Delimiter . -Header $Header; + $P | % { $_.Major = [int]$_.Major }; + $P | % { $_.Minor = [int]$_.Minor }; + $P = $P | Sort -Property Major,Minor,Build,Revision; + #$P | Format-Table; + + $Q = $P | % { ($_.PSObject.Properties | % { $_.Value }) -join '.'}; + + return $Q.TrimEnd("."); +} + +"versionToUnlist: " + $versionToUnlist; +"unlistOlderVersionsAlso: " + $unlistOlderVersionsAlso; +"unlistPackagesForReal: " + $unlistPackagesForReal; +" "; +"Package versions to unlist:" + +$result = Invoke-RestMethod -Uri $feedStateUrl -Method Get -ContentType "application/json"; + +foreach ($packageName in $packageNames) { + $versionsToUnlist = $Null; + $index = -1; + + $packageInfo = $result.packages | Where-Object {$_.id -eq $packageName}; + + [string]$unsortedVersions = $packageInfo.versions | %{ $_ + "`r`n" }; + + $sortedVersions = Sort-Versions $unsortedVersions; + + if ($unlistOlderVersionsAlso -eq "true") { + for ([int]$i = 0; $i -lt $sortedVersions.Count; $i++) + { + if ($sortedVersions[$i] -ge $versionToUnlist) { + $index = $i; + if ($sortedVersions[$i] -gt $versionToUnlist) { $index--; } + break; + } + } + + if ($index -ne $Null -and $index -ge 0) { + [string[]]$versionsToUnlist = $sortedVersions | select -First ($index + 1); + } + } else { + [string[]]$versionsToUnlist = $sortedVersions.Where({$_ -eq $versionToUnlist}); + } + + if ($versionsToUnlist.Count -gt 0) { + "-----------------------------------------"; + $packageName + ":"; + "-----------------------------------------"; + + foreach ($version in $versionsToUnlist) { + if ($unlistPackagesForReal -eq "true") { + " Unlisting $version"; + " nuget delete $packageName $version -Source $feedApiUrl -apikey $myGetPersonalAccessToken -NonInteractive"; + nuget delete $packageName $version -Source $feedApiUrl -apikey $myGetPersonalAccessToken -NonInteractive; + } else { + " $version"; + } + } + } +} +"-----------------------------------------"; \ No newline at end of file diff --git a/build/UnlistPackagesOnNpm.ps1 b/build/UnlistPackagesOnNpm.ps1 new file mode 100644 index 000000000..a10ef704e --- /dev/null +++ b/build/UnlistPackagesOnNpm.ps1 @@ -0,0 +1,74 @@ +# +# Unlists package versions on npm feed lower than or equal to $versionToUnlist. +# Run this first with $unlistPackagesForReal = false (default) to verify what versions will be affected. +# +param +( + [string]$versionToUnlist = "0.6.2", + [string]$unlistOlderVersionsAlso = "false", + [string[]]$packageNames = @( "adaptive-expressions","botbuilder","botbuilder-dialogs-adaptive-runtime-core" ), + [string]$npmPersonalAccessToken, # Not currently used. + [string]$unlistPackagesForReal = "false" +) + +$RegistryUrlSource = "https://pkgs.dev.azure.com/ConversationalAI/BotFramework/_packaging/SDK/npm/registry/"; + +Function Get-Versions-From-Npm +{ + param ( [string]$packageName ); + + $result = npm view $packageName versions + $versions = $result | ConvertFrom-Json; + + return $versions; +} + +"versionToUnlist: " + $versionToUnlist; +"unlistOlderVersionsAlso: " + $unlistOlderVersionsAlso; +"unlistPackagesForReal: " + $unlistPackagesForReal; +" "; +"Package versions to unlist:" + +foreach ($packageName in $packageNames) { + $versionsToUnlist = $Null; + $index = -1; + + $versions = Get-Versions-From-Npm $packageName; + $sortedVersions = $versions -split " " | Where-Object {$_}; + + if ($unlistOlderVersionsAlso -eq "true") { + for ([int]$i = 0; $i -lt $sortedVersions.Count; $i++) + { + if ($sortedVersions[$i] -ge $versionToUnlist) { + $index = $i; + if ($sortedVersions[$i] -gt $versionToUnlist) { $index--; } + break; + } + } + + if ($index -ne $Null -and $index -ge 0) { + [string[]]$versionsToUnlist = $sortedVersions | select -First ($index + 1); + } + } else { + [string[]]$versionsToUnlist = $sortedVersions.Where({$_ -eq $versionToUnlist}); + } + + if ($versionsToUnlist.Count -gt 0) { + "-----------------------------------------"; + $packageName + ":"; + "-----------------------------------------"; + + foreach ($version in $versionsToUnlist) { + if ($unlistPackagesForReal -eq "true") { + " Unlisting $version"; + " npm unpublish $packageName@$version --loglevel verbose"; + npm unpublish $packageName@$version --loglevel verbose; + #"npm unpublish @microsoft/$packageName@$version --loglevel verbose"; + #npm unpublish @microsoft/$packageName@$version --loglevel verbose; + } else { + " $version"; + } + } + } +} +"-----------------------------------------"; diff --git a/build/UnlistPackagesOnNuGet.ps1 b/build/UnlistPackagesOnNuGet.ps1 new file mode 100644 index 000000000..a2cd987eb --- /dev/null +++ b/build/UnlistPackagesOnNuGet.ps1 @@ -0,0 +1,76 @@ +# +# Unlists NuGet packages on NuGet.org with the specified version number. Option to unlist all older versions as well. +# Run this first with $unlistPackagesForReal = false (default) to verify what versions will be affected. +# See: https://stackoverflow.com/questions/34958908/where-can-i-find-documentation-for-the-nuget-v3-api +# +param +( + [string]$versionToUnlist = "1.0.2", + [string]$unlistOlderVersionsAlso = "false", + [string[]]$packageNames = @( "AdaptiveExpressions","Microsoft.Bot.Builder","Microsoft.Bot.Builder.Integration.AspNet.Core" ), + [string]$nuGetPersonalAccessToken, + [string]$unlistPackagesForReal = "false" +) + +$feedApiUrl = "https://api.nuget.org/v3/index.json"; + +Function Get-Versions-From-Nuget +{ + param ( [string]$packageName ); + + $packageBaseAddress = "https://api.nuget.org/v3-flatcontainer/"; + [string]$versionsUri = $packageBaseAddress + $packageName + "/index.json"; + + $response2 = Invoke-RestMethod -Uri $versionsUri + + $versions = $response2.versions; + + return $versions; +} + +"versionToUnlist: " + $versionToUnlist; +"unlistOlderVersionsAlso: " + $unlistOlderVersionsAlso; +"unlistPackagesForReal: " + $unlistPackagesForReal; +" "; +"Package versions to unlist:" + +foreach ($packageName in $packageNames) { + $versionsToUnlist = $Null; + $index = -1; + + $sortedVersions = Get-Versions-From-Nuget $packageName; + + if ($unlistOlderVersionsAlso -eq "true") { + for ([int]$i = 0; $i -lt $sortedVersions.Count; $i++) + { + if ($sortedVersions[$i] -ge $versionToUnlist) { + $index = $i; + if ($sortedVersions[$i] -gt $versionToUnlist) { $index--; } + break; + } + } + + if ($index -ne $Null -and $index -ge 0) { + [string[]]$versionsToUnlist = $sortedVersions | select -First ($index + 1); + } + } else { + [string[]]$versionsToUnlist = $sortedVersions.Where({$_ -eq $versionToUnlist}); + } + + if ($versionsToUnlist.Count -gt 0) { + "-----------------------------------------"; + $packageName + ":"; + "-----------------------------------------"; + + foreach ($version in $versionsToUnlist) { + if ($unlistPackagesForReal -eq "true") { + " Unlisting $version"; + " nuget delete $packageName $version -Source $feedApiUrl -apikey $nuGetPersonalAccessToken -NonInteractive"; + nuget delete $packageName $version -Source $feedApiUrl -apikey $nuGetPersonalAccessToken -NonInteractive; + } else { + " $version"; + } + } + } +} +"-----------------------------------------";