[vcpkg-artifacts] Fix end to end development. (#586)

* [vcpkg-artifacts] Fix end to end development. Thanks to @fearthecowboy for help on this one. I considered ripping out the rush dependency, since it seems like we are only ever going to have one meaningful "project", and the dependency deduplication we need seems to be already done by pnpm (rather than npm), but after talking with @fearthecowboy I've decided to not go there since we still have rush linking the test project in. Unfortunately, there does not appear to be an effiicent way to build the typescript parts out-of-source, since they depend on node_modules which is put into the source tree. This adds a vcpkg.ps1 which does the same "environment hacking" as the in-development ce.ps1, teaches CMakeLists.txt to invoke rush and the typescript compiler as necessary, and teaches vcpkg.exe to use a hard-coded-into-the-binary path to the source tree when that in-development setting is turned on. The previous "always download latest ce bits" behavior is retained for folks who build vcpkg.exe from source and don't want to arrange for node and rush to be available. * format * Add better messages during build. * Guard VCPKG_ARTIFACTS_DEVELOPMENT for a recent enough CMake to not trigger infinite loops. * Fix initial call to rush rebuild needed to make incremental build work afterwards. * Fix generating vcpkg.ps1 on Linux. * Add prerequisites instructions. * Make ps1 executable. * Add dependencies on the node_modules directory. * Use ALL instead of add_dependencies.
2022-06-24 13:51:58 -07:00 · 2022-06-24 13:51:58 -07:00 · 426593ed26
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -19,6 +19,7 @@ option(VCPKG_BUILD_TLS12_DOWNLOADER "Enable building the tls12-downloader" OFF)
 option(VCPKG_BUILD_FUZZING "Option for enabling vcpkg-fuzz support" OFF)
 option(VCPKG_EMBED_GIT_SHA "Option for to fill in the Git SHA version; off by default to avoid privacy concerns out of official builds" OFF)
 option(VCPKG_ADD_SOURCELINK "Option for enabling SourceLink in debug information on Windows/MSVC builds" "${VCPKG_EMBED_GIT_SHA}")
+option(VCPKG_ARTIFACTS_DEVELOPMENT "Hard code path to artifacts TypeScript. Requires node.js and global install of @microsoft/rush." OFF)
 option(VCPKG_OFFICIAL_BUILD "Option to cause immediate failure if variables required for official builds are unset." OFF)
 set(VCPKG_PDB_SUFFIX "" CACHE STRING "Append this string to the name of the PDB for shipping vcpkg binaries.")

@ -39,6 +40,11 @@ if(VCPKG_DEVELOPMENT_WARNINGS)
    set(FMT_PEDANTIC ON CACHE BOOL "")
 endif()

+if (VCPKG_ARTIFACTS_DEVELOPMENT)
+    # https://gitlab.kitware.com/cmake/cmake/-/issues/20245
+    cmake_minimum_required(VERSION 3.17)
+endif()
+
 project(vcpkg
    DESCRIPTION "vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS."
    HOMEPAGE_URL "https://github.com/microsoft/vcpkg"
@ -170,7 +176,95 @@ if(NOT DEFINED VCPKG_CE_SHA OR VCPKG_CE_SHA STREQUAL "")
    if(VCPKG_OFFICIAL_BUILD)
        message(FATAL_ERROR "VCPKG_CE_SHA is required for official builds.")
    endif()
+    if (VCPKG_ARTIFACTS_DEVELOPMENT)
+        file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce" VCPKG_ARTIFACTS_PATH)
+        target_compile_definitions(vcpkglib PUBLIC "VCPKG_ARTIFACTS_PATH=${VCPKG_ARTIFACTS_PATH}")
+        mark_as_advanced(VCPKG_ARTIFACTS_PATH)
+
+        if (WIN32)
+            set(RUSH_SUFFIX ".cmd")
        else()
+            set(RUSH_SUFFIX "")
+        endif()
+
+        find_program(NODEJS "node")
+        if (NOT NODEJS)
+            message(FATAL_ERROR "node.js and @microsoft/rush must be installed when VCPKG_ARTIFACTS_DEVELOPMENT is set")
+        endif()
+
+        find_program(RUSH "rush${RUSH_SUFFIX}")
+        if (NOT RUSH)
+            message(FATAL_ERROR "@microsoft/rush is required when VCPKG_ARTIFACTS_DEVELOPMENT is set; use `npm install -g @microsoft/rush`")
+        endif()
+
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+            COMMAND "${RUSH}" ARGS "update"
+            COMMAND "${RUSH}" ARGS "rebuild"
+            WORKING_DIRECTORY
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce"
+            COMMENT
+                "Running rush update..."
+            VERBATIM
+            )
+
+        add_custom_target(rush-update
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+                )
+
+# === Target: vcpkg-artifacts ===
+        file(GLOB_RECURSE VCPKG_ARTIFACTS_SOURCES CONFIGURE_DEPENDS "ce/ce/*.ts")
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_LIST_DIR}/ce/ce/dist/tsconfig.tsbuildinfo"
+            COMMAND
+                "${NODEJS}" "${CMAKE_CURRENT_LIST_DIR}/ce/ce/node_modules/typescript/bin/tsc"
+                    -p "${CMAKE_CURRENT_LIST_DIR}/ce/ce"
+            DEPENDS
+                ${VCPKG_ARTIFACTS_SOURCES}
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+            COMMENT
+                "Building vcpkg-ce..."
+            VERBATIM
+            )
+
+        add_custom_target(vcpkg-artifacts
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_LIST_DIR}/ce/ce/dist/tsconfig.tsbuildinfo"
+            )
+
+# === Target: vcpkg-artifacts-test ===
+        file(GLOB_RECURSE VCPKG_ARTIFACTS_TEST_SOURCES CONFIGURE_DEPENDS "ce/test/*.ts")
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_LIST_DIR}/ce/test/dist/tsconfig.tsbuildinfo"
+            COMMAND
+                "${NODEJS}" "${CMAKE_CURRENT_LIST_DIR}/ce/test/node_modules/typescript/bin/tsc"
+                    -p "${CMAKE_CURRENT_LIST_DIR}/ce/test"
+            DEPENDS
+                ${VCPKG_ARTIFACTS_TEST_SOURCES}
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+            COMMENT
+                "Building vcpkg-ce-test..."
+            VERBATIM
+            )
+
+        add_custom_target(vcpkg-artifacts-test
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_LIST_DIR}/ce/test/dist/tsconfig.tsbuildinfo"
+            )
+    endif()
+else()
+    if (VCPKG_ARTIFACTS_DEVELOPMENT)
+        message(WARNING "VCPKG_CE_SHA overrides VCPKG_ARTIFACTS_DEVELOPMENT")
+    endif()
    target_compile_definitions(vcpkglib PUBLIC
        VCPKG_CE_SHA=${VCPKG_CE_SHA}
    )
@ -240,9 +334,21 @@ if(MINGW)
 endif()

 # === Target: vcpkg ===
+add_custom_command(
+    OUTPUT
+        "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1"
+    COMMAND
+        "${CMAKE_COMMAND}" ARGS -E copy "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg-in-development.ps1" "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1"
+    DEPENDS
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg-in-development.ps1"
+    VERBATIM
+    )
+
+add_custom_target(vcpkg-ps1 ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1")

 add_executable(vcpkg ${VCPKG_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
 target_link_libraries(vcpkg PRIVATE vcpkglib)
+
 vcpkg_target_add_warning_options(vcpkg)
 if(VCPKG_ADD_SOURCELINK)
    if(VCPKG_VERSION STREQUAL "unknownhash")
--- a/CMakeSettings.json
+++ b/CMakeSettings.json
@ -0,0 +1,136 @@
+{
+  "configurations": [
+    {
+      "name": "x64-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x64-Release",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x86-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x86_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x86-Release",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "msvc_x86_x64" ],
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    }
+  ]
+}
--- a/README.md
+++ b/README.md
@ -75,6 +75,21 @@ with any additional questions or comments.
 [contributing:coc]: https://opensource.microsoft.com/codeofconduct/
 [contributing:coc-faq]: https://opensource.microsoft.com/codeofconduct/

+## Windows Contributing Prerequisites
+
+* Install Visual Studio and the C++ workload
+* Install Node.JS by downloading a 16.x copy from https://nodejs.org/en/
+* `npm install -g @microsoft/rush`
+
+## Ubuntu 22.04 Contributing Prerequisites
+
+```
+curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+sudo apt update
+sudo apt install nodejs cmake ninja-build gcc build-essential git zip unzip
+sudo npm install -g @microsoft/rush
+```
+
 # License

 The product code in this repository is licensed under the [MIT License](LICENSE.txt). The tests
--- a/ce/ce/tsconfig.json
+++ b/ce/ce/tsconfig.json
@ -2,11 +2,7 @@
  "extends": "../common/tsconfig.json",
  "compilerOptions": {
    "outDir": "dist",
-    "rootDir": ".",
-    "types": [
-      "node"
-    ],
-    "inlineSourceMap": true
+    "rootDir": "."
  },
  "include": [
    "./**/*.ts"
--- a/ce/common/tsconfig.json
+++ b/ce/common/tsconfig.json
@ -24,7 +24,8 @@
    ],
    "experimentalDecorators": true,
    "emitDecoratorMetadata": true,
-    "newLine": "LF"
+    "newLine": "LF",
+    "incremental": true
  },
  "exclude": [
    "../**/dist/**",
--- a/ce/test/tsconfig.json
+++ b/ce/test/tsconfig.json
@ -1,14 +1,12 @@
 {
  "extends": "../common/tsconfig.json",
  "compilerOptions": {
-    "experimentalDecorators": true,
    "outDir": "./dist",
    "rootDir": ".",
    "types": [
      "node",
      "mocha"
-    ],
-    "inlineSourceMap": true,
+    ]
  },
  "include": [
    "./**/*.ts"
--- a/src/vcpkg-in-development.ps1
+++ b/src/vcpkg-in-development.ps1
@ -0,0 +1,45 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+$ENV:NODE_OPTIONS="--enable-source-maps"
+
+
+function resolve {
+    param ( [string] $name )
+    $name = Resolve-Path $name -ErrorAction 0 -ErrorVariable _err
+    if (-not($name)) { return $_err[0].TargetObject }
+    $Error.clear()
+    return $name
+}
+
+
+if( $ENV:VCPKG_ROOT ) {
+  $SCRIPT:VCPKG_ROOT=(resolve $ENV:VCPKG_ROOT)
+  $ENV:VCPKG_ROOT=$VCPKG_ROOT
+} else {
+  $SCRIPT:VCPKG_ROOT=(resolve "$HOME/.vcpkg")
+  $ENV:VCPKG_ROOT=$VCPKG_ROOT
+}
+
+# setup the postscript file
+# Generate 31 bits of randomness, to avoid clashing with concurrent executions.
+$env:Z_VCPKG_POSTSCRIPT = resolve "${VCPKG_ROOT}/VCPKG_tmp_$(Get-Random -SetSeed $PID).ps1"
+
+[string]$vcpkgPath = "$PSScriptRoot/vcpkg"
+if ($IsWindows) {
+  $vcpkgPath += ".exe"
+}
+
+& $vcpkgPath @args 
+
+# dot-source the postscript file to modify the environment
+if ($env:Z_VCPKG_POSTSCRIPT -and (Test-Path $env:Z_VCPKG_POSTSCRIPT)) {
+  # write-host (get-content -raw $env:Z_VCPKG_POSTSCRIPT)
+  $content = get-content -raw $env:Z_VCPKG_POSTSCRIPT
+
+  if( $content ) {
+    iex $content
+  }
+  Remove-Item -Force $env:Z_VCPKG_POSTSCRIPT
+  remove-item -ea 0 -force env:Z_VCPKG_POSTSCRIPT
+}
--- a/src/vcpkg/configure-environment.cpp
+++ b/src/vcpkg/configure-environment.cpp
@ -32,6 +32,7 @@ namespace
                                 "This message is normally displayed only in development.",
                                 "Downloading latest vcpkg-ce bundle...");

+#if !defined(VCPKG_ARTIFACTS_PATH)
    void extract_ce_tarball(const VcpkgPaths& paths,
                            const Path& ce_tarball,
                            const Path& node_path,
@ -64,6 +65,7 @@ namespace
            Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgFailedToProvisionCe);
        }
    }
+#endif // ^^^ !defined(VCPKG_ARTIFACTS_PATH)
 }

 namespace vcpkg
@ -102,7 +104,15 @@ namespace vcpkg
            extract_ce_tarball(paths, ce_tarball, node_path, node_modules);
            fs.write_contents(ce_sha_path, VCPKG_CE_SHA_AS_STRING, VCPKG_LINE_INFO);
        }
-#else  // ^^^ VCPKG_CE_SHA / !VCPKG_CE_SHA vvv
+#elif defined(VCPKG_ARTIFACTS_PATH)
+        // use hard coded in-source copy
+        (void)fs;
+        (void)download_manager;
+        ce_path = MACRO_TO_STRING(VCPKG_ARTIFACTS_PATH);
+        // development support: intentionally unlocalized
+        msg::println(Color::warning,
+                     LocalizedString::from_raw("Using in-development vcpkg-artifacts built at: ").append_raw(ce_path));
+#else  // ^^^ VCPKG_ARTIFACTS_PATH / give up and always download latest vvv
        fs.remove(ce_sha_path, VCPKG_LINE_INFO);
        fs.remove_all(ce_path, VCPKG_LINE_INFO);
        msg::println(Color::warning, msgDownloadingVcpkgCeBundleLatest);