[vcpkg-artifacts] Fix end to end development. (#586)

* [vcpkg-artifacts] Fix end to end development. Thanks to @fearthecowboy for help on this one. I considered ripping out the rush dependency, since it seems like we are only ever going to have one meaningful "project", and the dependency deduplication we need seems to be already done by pnpm (rather than npm), but after talking with @fearthecowboy I've decided to not go there since we still have rush linking the test project in. Unfortunately, there does not appear to be an effiicent way to build the typescript parts out-of-source, since they depend on node_modules which is put into the source tree. This adds a vcpkg.ps1 which does the same "environment hacking" as the in-development ce.ps1, teaches CMakeLists.txt to invoke rush and the typescript compiler as necessary, and teaches vcpkg.exe to use a hard-coded-into-the-binary path to the source tree when that in-development setting is turned on. The previous "always download latest ce bits" behavior is retained for folks who build vcpkg.exe from source and don't want to arrange for node and rush to be available. * format * Add better messages during build. * Guard VCPKG_ARTIFACTS_DEVELOPMENT for a recent enough CMake to not trigger infinite loops. * Fix initial call to rush rebuild needed to make incremental build work afterwards. * Fix generating vcpkg.ps1 on Linux. * Add prerequisites instructions. * Make ps1 executable. * Add dependencies on the node_modules directory. * Use ALL instead of add_dependencies.
2022-06-24 13:51:58 -07:00 · 2022-06-24 13:51:58 -07:00 · 426593ed26
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -19,6 +19,7 @@ option(VCPKG_BUILD_TLS12_DOWNLOADER "Enable building the tls12-downloader" OFF)
 option(VCPKG_BUILD_FUZZING "Option for enabling vcpkg-fuzz support" OFF)
 option(VCPKG_EMBED_GIT_SHA "Option for to fill in the Git SHA version; off by default to avoid privacy concerns out of official builds" OFF)
 option(VCPKG_ADD_SOURCELINK "Option for enabling SourceLink in debug information on Windows/MSVC builds" "${VCPKG_EMBED_GIT_SHA}")
+option(VCPKG_ARTIFACTS_DEVELOPMENT "Hard code path to artifacts TypeScript. Requires node.js and global install of @microsoft/rush." OFF)
 option(VCPKG_OFFICIAL_BUILD "Option to cause immediate failure if variables required for official builds are unset." OFF)
 set(VCPKG_PDB_SUFFIX "" CACHE STRING "Append this string to the name of the PDB for shipping vcpkg binaries.")

@ -39,6 +40,11 @@ if(VCPKG_DEVELOPMENT_WARNINGS)
    set(FMT_PEDANTIC ON CACHE BOOL "")
 endif()

+if (VCPKG_ARTIFACTS_DEVELOPMENT)
+    # https://gitlab.kitware.com/cmake/cmake/-/issues/20245
+    cmake_minimum_required(VERSION 3.17)
+endif()
+
 project(vcpkg
    DESCRIPTION "vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS."
    HOMEPAGE_URL "https://github.com/microsoft/vcpkg"
@ -170,7 +176,95 @@ if(NOT DEFINED VCPKG_CE_SHA OR VCPKG_CE_SHA STREQUAL "")
    if(VCPKG_OFFICIAL_BUILD)
        message(FATAL_ERROR "VCPKG_CE_SHA is required for official builds.")
    endif()
+    if (VCPKG_ARTIFACTS_DEVELOPMENT)
+        file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce" VCPKG_ARTIFACTS_PATH)
+        target_compile_definitions(vcpkglib PUBLIC "VCPKG_ARTIFACTS_PATH=${VCPKG_ARTIFACTS_PATH}")
+        mark_as_advanced(VCPKG_ARTIFACTS_PATH)
+
+        if (WIN32)
+            set(RUSH_SUFFIX ".cmd")
+        else()
+            set(RUSH_SUFFIX "")
+        endif()
+
+        find_program(NODEJS "node")
+        if (NOT NODEJS)
+            message(FATAL_ERROR "node.js and @microsoft/rush must be installed when VCPKG_ARTIFACTS_DEVELOPMENT is set")
+        endif()
+
+        find_program(RUSH "rush${RUSH_SUFFIX}")
+        if (NOT RUSH)
+            message(FATAL_ERROR "@microsoft/rush is required when VCPKG_ARTIFACTS_DEVELOPMENT is set; use `npm install -g @microsoft/rush`")
+        endif()
+
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+            COMMAND "${RUSH}" ARGS "update"
+            COMMAND "${RUSH}" ARGS "rebuild"
+            WORKING_DIRECTORY
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce"
+            COMMENT
+                "Running rush update..."
+            VERBATIM
+            )
+
+        add_custom_target(rush-update
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+                )
+
+# === Target: vcpkg-artifacts ===
+        file(GLOB_RECURSE VCPKG_ARTIFACTS_SOURCES CONFIGURE_DEPENDS "ce/ce/*.ts")
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_LIST_DIR}/ce/ce/dist/tsconfig.tsbuildinfo"
+            COMMAND
+                "${NODEJS}" "${CMAKE_CURRENT_LIST_DIR}/ce/ce/node_modules/typescript/bin/tsc"
+                    -p "${CMAKE_CURRENT_LIST_DIR}/ce/ce"
+            DEPENDS
+                ${VCPKG_ARTIFACTS_SOURCES}
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/ce/node_modules"
+            COMMENT
+                "Building vcpkg-ce..."
+            VERBATIM
+            )
+
+        add_custom_target(vcpkg-artifacts
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_LIST_DIR}/ce/ce/dist/tsconfig.tsbuildinfo"
+            )
+
+# === Target: vcpkg-artifacts-test ===
+        file(GLOB_RECURSE VCPKG_ARTIFACTS_TEST_SOURCES CONFIGURE_DEPENDS "ce/test/*.ts")
+        add_custom_command(
+            OUTPUT
+                "${CMAKE_CURRENT_LIST_DIR}/ce/test/dist/tsconfig.tsbuildinfo"
+            COMMAND
+                "${NODEJS}" "${CMAKE_CURRENT_LIST_DIR}/ce/test/node_modules/typescript/bin/tsc"
+                    -p "${CMAKE_CURRENT_LIST_DIR}/ce/test"
+            DEPENDS
+                ${VCPKG_ARTIFACTS_TEST_SOURCES}
+                "${CMAKE_CURRENT_SOURCE_DIR}/ce/test/node_modules"
+            COMMENT
+                "Building vcpkg-ce-test..."
+            VERBATIM
+            )
+
+        add_custom_target(vcpkg-artifacts-test
+            ALL
+            DEPENDS
+                "${CMAKE_CURRENT_LIST_DIR}/ce/test/dist/tsconfig.tsbuildinfo"
+            )
+    endif()
 else()
+    if (VCPKG_ARTIFACTS_DEVELOPMENT)
+        message(WARNING "VCPKG_CE_SHA overrides VCPKG_ARTIFACTS_DEVELOPMENT")
+    endif()
    target_compile_definitions(vcpkglib PUBLIC
        VCPKG_CE_SHA=${VCPKG_CE_SHA}
    )
@ -240,9 +334,21 @@ if(MINGW)
 endif()

 # === Target: vcpkg ===
+add_custom_command(
+    OUTPUT
+        "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1"
+    COMMAND
+        "${CMAKE_COMMAND}" ARGS -E copy "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg-in-development.ps1" "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1"
+    DEPENDS
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg-in-development.ps1"
+    VERBATIM
+    )
+
+add_custom_target(vcpkg-ps1 ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/vcpkg.ps1")

 add_executable(vcpkg ${VCPKG_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
 target_link_libraries(vcpkg PRIVATE vcpkglib)
+
 vcpkg_target_add_warning_options(vcpkg)
 if(VCPKG_ADD_SOURCELINK)
    if(VCPKG_VERSION STREQUAL "unknownhash")
--- a/CMakeSettings.json
+++ b/CMakeSettings.json
@ -0,0 +1,136 @@
+{
+  "configurations": [
+    {
+      "name": "x64-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x64-Release",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x86-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x86_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    },
+    {
+      "name": "x86-Release",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "inheritEnvironments": [ "msvc_x86_x64" ],
+      "variables": [
+        {
+          "name": "VCPKG_ARTIFACTS_DEVELOPMENT",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_TLS12_DOWNLOADER",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_BENCHMARKING",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "VCPKG_BUILD_FUZZING",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ]
+    }
+  ]
+}
--- a/README.md
+++ b/README.md
@ -19,15 +19,15 @@ This repository contains the contents formerly at https://github.com/microsoft/v
 # Vcpkg-ce: "Configure Environment" / artifacts

 Parts of vcpkg powered by "ce" are currently in 'preview' -- there will most certainly be changes between now
-and when the tool is 'released' based on feedback. 
+and when the tool is 'released' based on feedback.

-You can use it, but be forewarned that we may change formats, commands, etc. 
+You can use it, but be forewarned that we may change formats, commands, etc.

-Think of it as a manifest-driven desired state configuration for C/C++ projects. 
+Think of it as a manifest-driven desired state configuration for C/C++ projects.

-It 
+It
 - integrates itself into your shell (PowerShell, CMD, bash/zsh)
- - can restore artifacts according to a manifest that follows one’s code 
+ - can restore artifacts according to a manifest that follows one’s code
 - provides discoverability interfaces

 ## Installation
@ -54,7 +54,7 @@ While the usage of `ce` is the same on all platforms, the installation/loading/r
 | `artifact metadata` | A description of the locations one or more artifacts describing rules for which ones are deployed given selection of a host architecture, target architecture, or other properties|
 | `artifact identity` | A short string that uniquely describes a moniker that a given artifact (and its metadata) can be referenced by. They can have one of the following forms:<br> `full/identity/path` - the full identity of an artifact that is in the built-in artifact source<br>`sourcename:full/identity/path` - the full identity of an artifact that is in the artifact source specified by the sourcename prefix<br>`shortname` - the shortened unique name of an artifact that is in the built-in artifact source<br>`sourcename:shortname` - the shortened unique name of an artifact that is in the artifact source specified by the sourcename prefix<br>Shortened names are generated based off the shortest unique identity path in the given source. |
 | `artifact source` | Also known as a “feed”. An Artifact Source is a location that hosts metadata to locate artifacts. (_There is only one source currently_) |
-| `project profile` | The per-project configuration file (`environment.yaml` or `environment.json`) 
+| `project profile` | The per-project configuration file (`environment.yaml` or `environment.json`)
 | `AMF`&nbsp;or&nbsp;`Metadata`&nbsp;`Format` | The schema / format of the YAML/JSON files for project profiles, global settings, and artifacts metadata. |
 | `activation` | The process by which a particular set of artifacts are acquired and enabled for use in a calling command program.|
 | `versions` | Version numbers are specified using the Semver format. If a version for a particular operation isn't specified, a range for the latest version ( `*` ) is assumed. A version or version range can be specified using the npm semver matching syntax. When a version is stored, it can be stored using the version range specified, a space and then the version found. (ie, the first version is what was asked for, the second is what was installed. No need for a separate lock file.) |
@ -75,6 +75,21 @@ with any additional questions or comments.
 [contributing:coc]: https://opensource.microsoft.com/codeofconduct/
 [contributing:coc-faq]: https://opensource.microsoft.com/codeofconduct/

+## Windows Contributing Prerequisites
+
+* Install Visual Studio and the C++ workload
+* Install Node.JS by downloading a 16.x copy from https://nodejs.org/en/
+* `npm install -g @microsoft/rush`
+
+## Ubuntu 22.04 Contributing Prerequisites
+
+```
+curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+sudo apt update
+sudo apt install nodejs cmake ninja-build gcc build-essential git zip unzip
+sudo npm install -g @microsoft/rush
+```
+
 # License

 The product code in this repository is licensed under the [MIT License](LICENSE.txt). The tests
@ -82,8 +97,8 @@ contain 3rd party code as documented in `NOTICE.txt`.

 # Trademarks

-This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
-trademarks or logos is subject to and must follow 
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
+trademarks or logos is subject to and must follow
 [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
 Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
 Any use of third-party trademarks or logos are subject to those third-party's policies.
--- a/ce/ce/tsconfig.json
+++ b/ce/ce/tsconfig.json
@ -2,11 +2,7 @@
  "extends": "../common/tsconfig.json",
  "compilerOptions": {
    "outDir": "dist",
-    "rootDir": ".",
-    "types": [
-      "node"
-    ],
-    "inlineSourceMap": true
+    "rootDir": "."
  },
  "include": [
    "./**/*.ts"
--- a/ce/common/tsconfig.json
+++ b/ce/common/tsconfig.json
@ -24,7 +24,8 @@
    ],
    "experimentalDecorators": true,
    "emitDecoratorMetadata": true,
-    "newLine": "LF"
+    "newLine": "LF",
+    "incremental": true
  },
  "exclude": [
    "../**/dist/**",
--- a/ce/test/tsconfig.json
+++ b/ce/test/tsconfig.json
@ -1,14 +1,12 @@
 {
  "extends": "../common/tsconfig.json",
  "compilerOptions": {
-    "experimentalDecorators": true,
    "outDir": "./dist",
    "rootDir": ".",
    "types": [
      "node",
      "mocha"
-    ],
-    "inlineSourceMap": true,
+    ]
  },
  "include": [
    "./**/*.ts"
--- a/src/vcpkg-in-development.ps1
+++ b/src/vcpkg-in-development.ps1
@ -0,0 +1,45 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+$ENV:NODE_OPTIONS="--enable-source-maps"
+
+
+function resolve {
+    param ( [string] $name )
+    $name = Resolve-Path $name -ErrorAction 0 -ErrorVariable _err
+    if (-not($name)) { return $_err[0].TargetObject }
+    $Error.clear()
+    return $name
+}
+
+
+if( $ENV:VCPKG_ROOT ) {
+  $SCRIPT:VCPKG_ROOT=(resolve $ENV:VCPKG_ROOT)
+  $ENV:VCPKG_ROOT=$VCPKG_ROOT
+} else {
+  $SCRIPT:VCPKG_ROOT=(resolve "$HOME/.vcpkg")
+  $ENV:VCPKG_ROOT=$VCPKG_ROOT
+}
+
+# setup the postscript file
+# Generate 31 bits of randomness, to avoid clashing with concurrent executions.
+$env:Z_VCPKG_POSTSCRIPT = resolve "${VCPKG_ROOT}/VCPKG_tmp_$(Get-Random -SetSeed $PID).ps1"
+
+[string]$vcpkgPath = "$PSScriptRoot/vcpkg"
+if ($IsWindows) {
+  $vcpkgPath += ".exe"
+}
+
+& $vcpkgPath @args 
+
+# dot-source the postscript file to modify the environment
+if ($env:Z_VCPKG_POSTSCRIPT -and (Test-Path $env:Z_VCPKG_POSTSCRIPT)) {
+  # write-host (get-content -raw $env:Z_VCPKG_POSTSCRIPT)
+  $content = get-content -raw $env:Z_VCPKG_POSTSCRIPT
+
+  if( $content ) {
+    iex $content
+  }
+  Remove-Item -Force $env:Z_VCPKG_POSTSCRIPT
+  remove-item -ea 0 -force env:Z_VCPKG_POSTSCRIPT
+}
--- a/src/vcpkg/configure-environment.cpp
+++ b/src/vcpkg/configure-environment.cpp
@ -32,6 +32,7 @@ namespace
                                 "This message is normally displayed only in development.",
                                 "Downloading latest vcpkg-ce bundle...");

+#if !defined(VCPKG_ARTIFACTS_PATH)
    void extract_ce_tarball(const VcpkgPaths& paths,
                            const Path& ce_tarball,
                            const Path& node_path,
@ -64,6 +65,7 @@ namespace
            Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgFailedToProvisionCe);
        }
    }
+#endif // ^^^ !defined(VCPKG_ARTIFACTS_PATH)
 }

 namespace vcpkg
@ -102,7 +104,15 @@ namespace vcpkg
            extract_ce_tarball(paths, ce_tarball, node_path, node_modules);
            fs.write_contents(ce_sha_path, VCPKG_CE_SHA_AS_STRING, VCPKG_LINE_INFO);
        }
-#else  // ^^^ VCPKG_CE_SHA / !VCPKG_CE_SHA vvv
+#elif defined(VCPKG_ARTIFACTS_PATH)
+        // use hard coded in-source copy
+        (void)fs;
+        (void)download_manager;
+        ce_path = MACRO_TO_STRING(VCPKG_ARTIFACTS_PATH);
+        // development support: intentionally unlocalized
+        msg::println(Color::warning,
+                     LocalizedString::from_raw("Using in-development vcpkg-artifacts built at: ").append_raw(ce_path));
+#else  // ^^^ VCPKG_ARTIFACTS_PATH / give up and always download latest vvv
        fs.remove(ce_sha_path, VCPKG_LINE_INFO);
        fs.remove_all(ce_path, VCPKG_LINE_INFO);
        msg::println(Color::warning, msgDownloadingVcpkgCeBundleLatest);