Auto-update pylance and debugpy dependencies when running PreBuild.ps1 (#6848)

Build/PreBuild.ps1

@@ -1,9 +1,7 @@
-param ($pylanceTgz, $vstarget, $source, $outdir)
+param ($vstarget, $outdir, $pylanceVersion, $debugpyVersion)
 
 $ErrorActionPreference = "Stop"
 
-"Restoring Packages"
-
 # These packages require a versionless symlink pointing to the versioned install.
 $need_symlink = @(
     "python",
@@ -25,6 +23,14 @@ if (-not $vstarget) {
     $vstarget = "$vstarget.0"
 }
 
+if (-not $pylanceVersion) {
+    $pylanceVersion = "latest"
+}
+
+if (-not $debugpyVersion) {
+    $debugpyVersion = "latest"
+}
+
 $buildroot = $MyInvocation.MyCommand.Definition | Split-Path -Parent | Split-Path -Parent
 
 if (-not $outdir) {
@@ -40,6 +46,37 @@ $outdir = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPat
 
 Push-Location "$buildroot\Build"
 try {
+
+    # Install pylance version specified in package.json
+    # If this doesn't work, you probably need to set up your .npmrc file or you need permissions to the feed.
+    # See https://microsoft.sharepoint.com/teams/python/_layouts/15/Doc.aspx?sourcedoc=%7B30d33826-9f98-4d3e-890e-b7d198bbbcbe%7D&action=edit&wd=target(Python%20VS%2FDev%20Docs.one%7Cd7206ce2-cf40-437b-8ce9-1e55f4bc2f44%2FPylance%20in%20VS%7C6000d391-4e62-4a4d-89d2-7f7c1f005639%2F)&share=IgEmONMwmJ8-TYkOt9GYu7y-AeCM6R8r8Myty0Lj8CeOs4E
+    # Note that this will modify your package-lock.json file if the version was updated. This file should be committed into source control.
+    "Installing Pylance"
+
+    # overwrite the pylance version in the package.json with the specified version
+    $packageJsonFile = Join-Path $buildroot "package.json"
+    $packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json
+    # only overwrite if the values are different
+    if ($packageJson.devDependencies.'@pylance/pylance' -ne $pylanceVersion) {
+        $packageJson.devDependencies.'@pylance/pylance' = $pylanceVersion
+        # ConvertTo-Json has a default depth of 2, so make it bigger to avoid strange errors
+        $packageJson | ConvertTo-Json -depth 8 | Set-Content $packageJsonFile
+    }
+
+    # install pylance and update the package-lock.json file
+    npm install --save
+
+    # print out the installed version
+    $npmLsOutput = & npm ls @pylance/pylance
+    $installedPylanceVersion = $npmLsOutput[1] -split "@" | Select-Object -Last 1
+    "Installed Pylance $installedPylanceVersion"
+    
+    # add azdo build tag
+    # commenting this out for now since azdo is throwing errors for an unknown reason
+    #Write-Host "##vso[build.addbuildtag]Pylance-$installedPylanceVersion"
+
+    "-----"
+    "Restoring Packages"
     $arglist = "restore", "$vstarget\packages.config", "-OutputDirectory", "`"$outdir`"", "-Config", "nuget.config", "-NonInteractive"
     $nuget = Get-Command nuget.exe -EA 0
     if (-not $nuget) {
@@ -48,9 +85,9 @@ try {
     Start-Process -Wait -NoNewWindow $nuget.Source -ErrorAction Stop -ArgumentList $arglist
 
     $versions = @{}
-    ([xml](Get-Content "$vstarget\packages.config")).packages.package | ForEach-Object{ $versions[$_.id] = $_.version }
+    ([xml](Get-Content "$vstarget\packages.config")).packages.package | ForEach-Object { $versions[$_.id] = $_.version }
 
-    $need_symlink | Where-Object{ $versions[$_] } | ForEach-Object{
+    $need_symlink | Where-Object { $versions[$_] } | ForEach-Object {
         $existing = Get-Item "$outdir\$_" -EA 0
         if ($existing) {
             if ($existing.LinkType) {
@@ -64,12 +101,39 @@ try {
         New-Item -ItemType Junction "$outdir\$_" -Value "$outdir\$_.$($versions[$_])"
     } | Out-Null
 
-    $debugpyver = Get-Content "$buildroot\Build\debugpy-version.txt" -Raw 
-    Write-Host "Downloading debugpy version $debugpyver"
-    $debugpyarglist = "install_debugpy.py", $debugpyver, "`"$outdir`""
-    Start-Process -Wait -NoNewWindow "$outdir\python\tools\python.exe" -ErrorAction Stop -ArgumentList $debugpyarglist
+    # debugpy install must come after package restore because it uses python which is symlinked as part of the previous step
 
-    Write-Host "Updating Microsoft.Python.*.dll pdbs to be windows format"
+    "-----"
+    "Installing Debugpy"
+    # pip install python packaging utilities
+    # SilentlyContinue on error since pip warnings will cause the build to fail, and installing debugpy will fail later if this step fails anyway
+    $pipArgList = "-m", "pip", "--disable-pip-version-check", "install", "packaging" 
+    Start-Process -Wait -NoNewWindow "$outdir\python\tools\python.exe" -ErrorAction SilentlyContinue -ArgumentList $pipArgList
+
+    # install debugpy
+    $debugpyArglist = "install_debugpy.py", $debugpyVersion, "`"$outdir`""
+    Start-Process -Wait -NoNewWindow "$outdir\python\tools\python.exe" -ErrorAction Stop -ArgumentList $debugpyArglist
+
+    # print out the installed version
+    $installedDebugpyVersion = ""
+    $versionPyFile = Join-Path $outdir "debugpy\_version.py"
+    foreach ($line in Get-Content $versionPyFile) {
+        if ($line.Trim().StartsWith("`"version`"")) {
+            $installedDebugpyVersion = $line.split(":")[1].Trim(" `"") # trim spaces and double quotes
+            break
+        }
+    }
+    "Installed Debugpy $installedDebugpyVersion"
+
+    # write debugpy version out to $buildroot\build\debugpy-version.txt, since that file is used by Debugger.csproj and various other classes
+    Set-Content -NoNewline -Force -Path "$buildroot\build\debugpy-version.txt" -Value $installedDebugpyVersion
+
+    # add azdo build tag
+    # commenting this out for now since azdo is throwing errors for an unknown reason
+    #Write-Host "##vso[build.addbuildtag]Debugpy-$installedDebugpyVersion"
+
+    "-----"
+    "Updating Microsoft.Python.*.dll pdbs to be windows format"
     Get-ChildItem "$outdir\Microsoft.Python.Parsing\lib\netstandard2.0" -Filter "*.pdb" | ForEach-Object {
         # Skip if there's already a pdb2 file
         # Convert each pdb $_.FullName

Build/debugpy-version.txt

@@ -1 +1 @@
-1.4.3
+1.5.1

Build/install_debugpy.py

@@ -1,3 +1,5 @@
+# This is a slightly modified version of https://github.com/microsoft/vscode-python/blob/main/pythonFiles/install_debugpy.py
+
 import argparse
 import io
 import json
@@ -5,23 +7,24 @@ import os
 import urllib.request as url_lib
 import zipfile
 
-DEBUGGER_PACKAGE = "debugpy"
-DEBUGGER_PYTHON_VERSIONS = ("cp35", "cp36", "cp37", "cp38")
-DEBUGGER_EXCLUDED_PLATFORMS = ("manylinux", "macosx")
+# if this import fails, run PreBuild.ps1, which will pip install packaging
+from packaging.version import parse as version_parser
 
+DEBUGGER_PYTHON_VERSIONS = ("cp35", "cp36", "cp37", "cp38", "cp39")
+DEBUGGER_EXCLUDED_PLATFORMS = ("manylinux", "macosx")
 
 def _contains(s, parts=()):
     return any(p for p in parts if p in s)
 
-
+# Get json containing all debugpy package metadata and all releases
 def _get_package_data():
-    json_uri = "https://pypi.org/pypi/{0}/json".format(DEBUGGER_PACKAGE)
+    json_uri = "https://pypi.org/pypi/debugpy/json"
     # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
     # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
     with url_lib.urlopen(json_uri) as response:
         return json.loads(response.read())
 
-
+# Get the wheel url for a specific release
 def _get_debugger_wheel_urls(data, version):
     return list(
         r["url"]
@@ -29,36 +32,33 @@ def _get_debugger_wheel_urls(data, version):
         if _contains(r["url"], DEBUGGER_PYTHON_VERSIONS) and not _contains(r["url"], DEBUGGER_EXCLUDED_PLATFORMS)
     )
 
-
+# Download the wheel from the url and extract the files in the appropriate layout
 def _download_and_extract(root, url, version):
     root = os.getcwd() if root is None or root == "." else root
-    prefix = os.path.join("debugpy-{0}.data".format(version), "purelib")
+    #print(url)
     with url_lib.urlopen(url) as response:
-        # Extract only the contents of the purelib subfolder (parent folder of debugpy),
-        # since debugpy files rely on the presence of a 'debugpy' folder.
-        with zipfile.ZipFile(io.BytesIO(response.read()), "r") as wheel:
+        data = response.read()
+        with zipfile.ZipFile(io.BytesIO(data), "r") as wheel:
             for zip_info in wheel.infolist():
                 # Ignore dist info since we are merging multiple wheels
-                if ".dist-info" in zip_info.filename:
+                if ".dist-info/" in zip_info.filename:
                     continue
-                # Normalize path for Windows, the wheel folder structure
-                # uses forward slashes.
-                normalized = os.path.normpath(zip_info.filename)
-                # Flatten the folder structure.
-                zip_info.filename = normalized.split(prefix)[-1]
-                wheel.extract(zip_info, root)
-
+                #print("\t" + zip_info.filename)
+                wheel.extract(zip_info.filename, root)
 
 def main(root, ver):
     data = _get_package_data()
 
+    # if version is "latest", use the max version from the data
+    if (ver == "latest"):
+        ver = max(data["releases"].keys(), key=version_parser)
+
     for url in _get_debugger_wheel_urls(data, ver):
         _download_and_extract(root, url, ver)
 
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('version', help='Version of debugpy package.')
+    parser.add_argument('version', help='Version of debugpy package. If "latest" is specified, the latest release will be installed')
     parser.add_argument('outputdir', help='Output directory under which debugpy directory will be created.')
     args = parser.parse_args()
     main(args.outputdir, args.version)

Build/templates/configure_pylance.yml

@@ -16,17 +16,4 @@ steps:
   - powershell: 'Get-Content ".npmrc"'
     displayName: 'Print .npmrc contents'
 
-  # todo: if pylance version parameter is set to "latest", use npm to get latest version and overwrite
-  # value in package.json for this build only
-
-  # todo: if pylance version parameter is set to a specific value, overwrite value in package.json
-  # for this build only
-
-  # Run npm to install pylance version specified in package.json
-  - powershell: 'npm install --verbose'
-    displayName: 'Install pylance with npm'
-
-  # todo: add pylance version build tag, similar to the following:
-  #- powershell: 'Write-Host "##vso[build.addbuildtag]$env:VSTarget"'
-  #  displayName: 'Add vstarget build tag'
-  #  condition: notin(variables['Build.Reason'], 'PullRequest')
+  # pylance installation has been moved to the 'Restore packages' step

azure-pipelines.yml

@@ -1,6 +1,16 @@
 # This pipeline is used to build the PTVS product and installer.
 # A seperate release pipeline is used to create insertion PR's into Visual Studio.
 
+parameters:
+- name: pylanceVersion
+  displayName: Pylance Version
+  type: string
+  default: latest
+- name: debugpyVersion
+  displayName: Debugpy Version
+  type: string
+  default: latest
+
 # build number format 
 name: $(date:yy)$(DayOfYear)$(rev:.r)
 
@@ -75,16 +85,15 @@ steps:
   # install microbuild plugins used for swixproj/vsmanproj, signing, and localization
   - template: Build/templates/install_microbuild_plugins.yml
 
-  # install pylance so it gets packaged into the PTVS installer
-  # todo: pass in pylance version to install (either "latest" or literal value)
-  - template: Build/templates/install_pylance.yml
+# configure pylance before installing it
+- template: Build/templates/configure_pylance.yml
 
-# Restore packages and install debugpy
+# Restore packages and install dependencies (pylance, debugpy)
 - task: PowerShell@1
   displayName: 'Restore packages'
   inputs:
     scriptName: Build/PreBuild.ps1
-    arguments: '-vstarget $(VSTarget)'
+    arguments: '-vstarget $(VSTarget) -pylanceVersion ${{ parameters.pylanceVersion }} -debugpyVersion ${{ parameters.debugpyVersion }}'
 
 # Build and publish logs
 - template: Build/templates/build.yml

package-lock.json

@@ -4,9 +4,9 @@
   "lockfileVersion": 1,
   "dependencies": {
     "@pylance/pylance": {
-      "version": "2021.11.0",
-      "resolved": "https://pkgs.dev.azure.com/devdiv/_packaging/Pylance/npm/registry/@pylance/pylance/-/pylance-2021.11.0.tgz",
-      "integrity": "sha1-n0tnG9TB9bMTxKibiy9LH0b6DmQ=",
+      "version": "2022.1.0",
+      "resolved": "https://pkgs.dev.azure.com/devdiv/_packaging/Pylance/npm/registry/@pylance/pylance/-/pylance-2022.1.0.tgz",
+      "integrity": "sha1-YaKAsjTqbj5fhbZ5/d3CYQ5sPpg=",
       "dev": true
     }
   }

package.json

@@ -2,6 +2,6 @@
   "name": "ptvs",
   "private": true,
   "devDependencies": {
-    "@pylance/pylance": "2021.11.0"
+    "@pylance/pylance": "latest"
   }
 }