diff --git a/.github/workflows/scalar-functional-tests.yml b/.github/workflows/scalar-functional-tests.yml new file mode 100644 index 0000000000..7226054aeb --- /dev/null +++ b/.github/workflows/scalar-functional-tests.yml @@ -0,0 +1,220 @@ +name: Scalar Functional Tests + +env: + SCALAR_REPOSITORY: microsoft/scalar + SCALAR_REF: main + DEBUG_WITH_TMATE: false + SCALAR_TEST_SKIP_VSTS_INFO: true + +on: + push: + branches: [ vfs-*, tentative/vfs-* ] + pull_request: + branches: [ vfs-*, features/* ] + +jobs: + scalar: + name: "Scalar Functional Tests" + + strategy: + fail-fast: false + matrix: + # Order by runtime (in descending order) + os: [windows-2019, macos-11, ubuntu-20.04, ubuntu-22.04] + # Scalar.NET used to be tested using `features: [false, experimental]` + # But currently, Scalar/C ignores `feature.scalar` altogether, so let's + # save some electrons and run only one of them... + features: [ignored] + exclude: + # The built-in FSMonitor is not (yet) supported on Linux + - os: ubuntu-20.04 + features: experimental + - os: ubuntu-22.04 + features: experimental + runs-on: ${{ matrix.os }} + + env: + BUILD_FRAGMENT: bin/Release/netcoreapp3.1 + GIT_FORCE_UNTRACKED_CACHE: 1 + + steps: + - name: Check out Git's source code + uses: actions/checkout@v3 + + - name: Setup build tools on Windows + if: runner.os == 'Windows' + uses: git-for-windows/setup-git-for-windows-sdk@v1 + + - name: Provide a minimal `install` on Windows + if: runner.os == 'Windows' + shell: bash + run: | + test -x /usr/bin/install || + tr % '\t' >/usr/bin/install <<-\EOF + #!/bin/sh + + cmd=cp + while test $# != 0 + do + %case "$1" in + %-d) cmd="mkdir -p";; + %-m) shift;; # ignore mode + %*) break;; + %esac + %shift + done + + exec $cmd "$@" + EOF + + - name: Install build dependencies for Git (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext + + - name: Build and install Git + shell: bash + env: + NO_TCLTK: Yup + run: | + # We do require a VFS version + def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)" + test -n "$def_ver" + + # Ensure that `git version` reflects DEF_VER + case "$(git describe --match "v[0-9]*vfs*" HEAD)" in + ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this + *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);; + esac + + SUDO= + extra= + case "${{ runner.os }}" in + Windows) + extra=DESTDIR=/c/Progra~1/Git + cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH + ;; + Linux) + SUDO=sudo + extra=prefix=/usr + ;; + macOS) + SUDO=sudo + extra=prefix=/usr/local + ;; + esac + + $SUDO make -j5 $extra install + + - name: Ensure that we use the built Git and Scalar + shell: bash + run: | + type -p git + git version + case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac + type -p scalar + scalar version + case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac + + - name: Check out Scalar's source code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works. + path: scalar + repository: ${{ env.SCALAR_REPOSITORY }} + ref: ${{ env.SCALAR_REF }} + + - name: Setup .NET Core + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.x' + + - name: Install dependencies + run: dotnet restore + working-directory: scalar + env: + DOTNET_NOLOGO: 1 + + - name: Build + working-directory: scalar + run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS. + + - name: Setup platform (Linux) + if: runner.os == 'Linux' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Mac) + if: runner.os == 'macOS' + run: | + echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Windows) + if: runner.os == 'Windows' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV + echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV + + - name: Configure feature.scalar + run: git config --global feature.scalar ${{ matrix.features }} + + - id: functional_test + name: Functional test + timeout-minutes: 60 + working-directory: scalar + shell: bash + run: | + export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event" + export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf" + export GIT_TRACE2_EVENT_BRIEF=true + export GIT_TRACE2_PERF_BRIEF=true + mkdir -p "$TRACE2_BASENAME" + mkdir -p "$TRACE2_BASENAME/Event" + mkdir -p "$TRACE2_BASENAME/Perf" + git version --build-options + cd ../out + Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite + + - name: Force-stop FSMonitor daemons and Git processes (Windows) + if: runner.os == 'Windows' && (success() || failure()) + shell: bash + run: | + set -x + wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID + wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete + wmic process where "ExecutablePath Like '%git.exe'" delete + + - id: trace2_zip_unix + if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Unix) + shell: bash + working-directory: scalar + run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/ + + - id: trace2_zip_windows + if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Windows) + working-directory: scalar + run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }} + + - name: Archive Trace2 Logs + if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' ) + uses: actions/upload-artifact@v3 + with: + name: ${{ env.TRACE2_BASENAME }}.zip + path: scalar/${{ env.TRACE2_BASENAME }}.zip + retention-days: 3 + + # The GitHub Action `action-tmate` allows developers to connect to the running agent + # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2 + # environment in `C:\msys64`, therefore it can be slightly tricky to interact with + # Git for Windows, which runs a slightly incompatible MSYS2 runtime). + - name: action-tmate + if: env.DEBUG_WITH_TMATE == 'true' && failure() + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 5a6b0c5f2b..22b26eb5e7 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -806,3 +806,12 @@ core.abbrev:: If set to "no", no abbreviation is made and the object names are shown in their full length. The minimum length is 4. + +core.configWriteLockTimeoutMS:: + When processes try to write to the config concurrently, it is likely + that one process "wins" and the other process(es) fail to lock the + config file. By configuring a timeout larger than zero, Git can be + told to try to lock the config again a couple times within the + specified timeout. If the timeout is configure to zero (which is the + default), Git will fail immediately when the config is already + locked. diff --git a/Documentation/scalar.txt b/Documentation/scalar.txt index f33436c7f6..35f67801e9 100644 --- a/Documentation/scalar.txt +++ b/Documentation/scalar.txt @@ -8,7 +8,9 @@ scalar - A tool for managing large Git repositories SYNOPSIS -------- [verse] -scalar clone [--single-branch] [--branch ] [--full-clone] [] +scalar clone [--single-branch] [--branch ] [--full-clone] + [--local-cache-path ] [--cache-server-url ] [--[no-]src] + [] scalar list scalar register [] scalar unregister [] @@ -16,6 +18,7 @@ scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) scalar reconfigure [ --all | ] scalar diagnose [] scalar delete +scalar cache-server ( --get | --set | --list [] ) [] DESCRIPTION ----------- @@ -80,10 +83,24 @@ remote-tracking branch for the branch this option was used for the initial cloning. If the HEAD at the remote did not point at any branch when `--single-branch` clone was made, no remote-tracking branch is created. +--no-src:: + Skip adding a `src` directory within the target enlistment. + --[no-]full-clone:: A sparse-checkout is initialized by default. This behavior can be turned off via `--full-clone`. +--local-cache-path :: + Override the path to the local cache root directory; Pre-fetched objects + are stored into a repository-dependent subdirectory of that path. ++ +The default is `:\.scalarCache` on Windows (on the same drive as the +clone), and `~/.scalarCache` on macOS. + +--cache-server-url :: + Retrieve missing objects from the specified remote, which is expected to + understand the GVFS protocol. + List ~~~~ @@ -157,6 +174,27 @@ delete :: This subcommand lets you delete an existing Scalar enlistment from your local file system, unregistering the repository. +Cache-server +~~~~~~~~~~~~ + +cache-server ( --get | --set | --list [] ) []:: + This command lets you query or set the GVFS-enabled cache server used + to fetch missing objects. + +--get:: + This is the default command mode: query the currently-configured cache + server URL, if any. + +--list:: + Access the `gvfs/info` endpoint of the specified remote (default: + `origin`) to figure out which cache servers are available, if any. ++ +In contrast to the `--get` command mode (which only accesses the local +repository), this command mode triggers a request via the network that +potentially requires authentication. If authentication is required, the +configured credential helper is employed (see linkgit:git-credential[1] +for details). + SEE ALSO -------- linkgit:git-clone[1], linkgit:git-maintenance[1]. diff --git a/Makefile b/Makefile index af69ce95db..93aca6a708 100644 --- a/Makefile +++ b/Makefile @@ -2722,6 +2722,7 @@ GIT_OBJS += git.o .PHONY: git-objs git-objs: $(GIT_OBJS) +SCALAR_OBJS := json-parser.o SCALAR_OBJS += scalar.o .PHONY: scalar-objs scalar-objs: $(SCALAR_OBJS) @@ -2870,7 +2871,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) -scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS) +scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \ $(filter %.o,$^) $(LIBS) diff --git a/abspath.c b/abspath.c index 7a5f114382..058e638e98 100644 --- a/abspath.c +++ b/abspath.c @@ -12,7 +12,7 @@ int is_directory(const char *path) } /* removes the last path component from 'path' except if 'path' is root */ -static void strip_last_component(struct strbuf *path) +void strip_last_path_component(struct strbuf *path) { size_t offset = offset_1st_component(path->buf); size_t len = path->len; @@ -117,7 +117,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, continue; /* '.' component */ } else if (next.len == 2 && !strcmp(next.buf, "..")) { /* '..' component; strip the last path component */ - strip_last_component(resolved); + strip_last_path_component(resolved); continue; } @@ -169,7 +169,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, * strip off the last component since it will * be replaced with the contents of the symlink */ - strip_last_component(resolved); + strip_last_path_component(resolved); } /* diff --git a/cache.h b/cache.h index 1c689455b3..91e0b9947e 100644 --- a/cache.h +++ b/cache.h @@ -611,6 +611,30 @@ void set_git_work_tree(const char *tree); #define ALTERNATE_DB_ENVIRONMENT "GIT_ALTERNATE_OBJECT_DIRECTORIES" void setup_work_tree(void); + +/* + * discover_git_directory_reason() is similar to discover_git_directory(), + * except it returns an enum value instead. It is important to note that + * a zero-valued return here is actually GIT_DIR_NONE, which is different + * from discover_git_directory. + */ +enum discovery_result { + GIT_DIR_NONE = 0, + GIT_DIR_EXPLICIT, + GIT_DIR_DISCOVERED, + GIT_DIR_BARE, + /* these are errors */ + GIT_DIR_HIT_CEILING = -1, + GIT_DIR_HIT_MOUNT_POINT = -2, + GIT_DIR_INVALID_GITFILE = -3, + GIT_DIR_INVALID_OWNERSHIP = -4, + GIT_DIR_DISALLOWED_BARE = -5, + GIT_DIR_INVALID_FORMAT = -6, + GIT_DIR_CWD_FAILURE = -7, +}; +enum discovery_result discover_git_directory_reason(struct strbuf *commondir, + struct strbuf *gitdir); + /* * Find the commondir and gitdir of the repository that contains the current * working directory, without changing the working directory or other global @@ -619,8 +643,12 @@ void setup_work_tree(void); * both have the same result appended to the buffer. The return value is * either 0 upon success and non-zero if no repository was found. */ -int discover_git_directory(struct strbuf *commondir, - struct strbuf *gitdir); +static inline int discover_git_directory(struct strbuf *commondir, + struct strbuf *gitdir) +{ + return discover_git_directory_reason(commondir, gitdir) <= 0; +} + const char *setup_git_directory_gently(int *); const char *setup_git_directory(void); char *prefix_path(const char *prefix, int len, const char *path); diff --git a/config.c b/config.c index e42ed10135..a9e086f3c8 100644 --- a/config.c +++ b/config.c @@ -3293,6 +3293,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename, const char *value_pattern, unsigned flags) { + static unsigned long timeout_ms = ULONG_MAX; int fd = -1, in_fd = -1; int ret; struct lock_file lock = LOCK_INIT; @@ -3313,11 +3314,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename, if (!config_filename) config_filename = filename_buf = git_pathdup("config"); + if ((long)timeout_ms < 0 && + git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms)) + timeout_ms = 0; + /* * The lock serves a purpose in addition to locking: the new * contents of .git/config will be written into it. */ - fd = hold_lock_file_for_update(&lock, config_filename, 0); + fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0, + timeout_ms); if (fd < 0) { error_errno(_("could not lock config file %s"), config_filename); ret = CONFIG_NO_LOCK; diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 41193f844c..a60cc1c9b9 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -797,7 +797,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main) add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c) target_link_libraries(git-shell common-main) -add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c) +add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c) target_link_libraries(scalar common-main) if(CURL_FOUND) diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md new file mode 100644 index 0000000000..a14f78a996 --- /dev/null +++ b/contrib/scalar/docs/faq.md @@ -0,0 +1,51 @@ +Frequently Asked Questions +========================== + +Using Scalar +------------ + +### I don't want a sparse clone, I want every file after I clone! + +Run `scalar clone --full-clone ` to initialize your repo to include +every file. You can switch to a sparse-checkout later by running +`git sparse-checkout init --cone`. + +### I already cloned without `--full-clone`. How do I get everything? + +Run `git sparse-checkout disable`. + +Scalar Design Decisions +----------------------- + +There may be many design decisions within Scalar that are confusing at first +glance. Some of them may cause friction when you use Scalar with your existing +repos and existing habits. + +> Scalar has the most benefit when users design repositories +> with efficient patterns. + +For example: Scalar uses the sparse-checkout feature to limit the size of the +working directory within a large monorepo. It is designed to work efficiently +with monorepos that are highly componentized, allowing most developers to +need many fewer files in their daily work. + +### Why does `scalar clone` create a `/src` folder? + +Scalar uses a file system watcher to keep track of changes under this `src` folder. +Any activity in this folder is assumed to be important to Git operations. By +creating the `src` folder, we are making it easy for your build system to +create output folders outside the `src` directory. We commonly see systems +create folders for build outputs and package downloads. Scalar itself creates +these folders during its builds. + +Your build system may create build artifacts such as `.obj` or `.lib` files +next to your source code. These are commonly "hidden" from Git using +`.gitignore` files. Having such artifacts in your source tree creates +additional work for Git because it needs to look at these files and match them +against the `.gitignore` patterns. + +By following the `src` pattern Scalar tries to establish and placing your build +intermediates and outputs parallel with the `src` folder and not inside it, +you can help optimize Git command performance for developers in the repository +by limiting the number of files Git needs to consider for many common +operations. diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md new file mode 100644 index 0000000000..d512533032 --- /dev/null +++ b/contrib/scalar/docs/getting-started.md @@ -0,0 +1,109 @@ +Getting Started +=============== + +Registering existing Git repos +------------------------------ + +To add a repository to the list of registered repos, run `scalar register []`. +If `` is not provided, then the "current repository" is discovered from +the working directory by scanning the parent paths for a path containing a `.git` +folder, possibly inside a `src` folder. + +To see which repositories are currently tracked by the service, run +`scalar list`. + +Run `scalar unregister []` to remove the repo from this list. + +Creating a new Scalar clone +--------------------------------------------------- + +The `clone` verb creates a local enlistment of a remote repository using the +partial clone feature available e.g. on GitHub, or using the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md), +such as Azure Repos. + +``` +scalar clone [options] [] +``` + +Create a local copy of the repository at ``. If specified, create the `` +directory and place the repository there. Otherwise, the last section of the `` +will be used for ``. + +At the end, the repo is located at `/src`. By default, the sparse-checkout +feature is enabled and the only files present are those in the root of your +Git repository. Use `git sparse-checkout set` to expand the set of directories +you want to see, or `git sparse-checkout disable` to expand to all files. You +can explore the subdirectories outside your sparse-checkout specification using +`git ls-tree HEAD`. + +### Sparse Repo Mode + +By default, Scalar reduces your working directory to only the files at the +root of the repository. You need to add the folders you care about to build up +to your working set. + +* `scalar clone ` + * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**. +* `cd \src` +* At this point, your `src` directory only contains files that appear in your root + tree. No folders are populated. +* Set the directory list for your sparse-checkout using: + 1. `git sparse-checkout set ...` + 2. `git sparse-checkout set --stdin < dir-list.txt` +* Run git commands as you normally would. +* To fully populate your working directory, run `git sparse-checkout disable`. + +If instead you want to start with all files on-disk, you can clone with the +`--full-clone` option. To enable sparse-checkout after the fact, run +`git sparse-checkout init --cone`. This will initialize your sparse-checkout +patterns to only match the files at root. + +If you are unfamiliar with what directories are available in the repository, +then you can run `git ls-tree -d --name-only HEAD` to discover the directories +at root, or `git ls-tree -d --name-only HEAD ` to discover the directories +in ``. + +### Options + +These options allow a user to customize their initial enlistment. + +* `--full-clone`: If specified, do not initialize the sparse-checkout feature. + All files will be present in your `src` directory. This behaves very similar + to a Git partial clone in that blobs are downloaded on demand. However, it + will use the GVFS protocol to download all Git objects. + +* `--cache-server-url=`: If specified, set the intended cache server to + the specified ``. All object queries will use the GVFS protocol to this + `` instead of the origin remote. If the remote supplies a list of + cache servers via the `/gvfs/config` endpoint, then the `clone` command + will select a nearby cache server from that list. + +* `--branch=`: Specify the branch to checkout after clone. + +* `--local-cache-path=`: Use this option to override the path for the + local Scalar cache. If not specified, then Scalar will select a default + path to share objects with your other enlistments. On Windows, this path + is a subdirectory of `:\.scalarCache\`. On Mac, this path is a + subdirectory of `~/.scalarCache/`. The default cache path is recommended so + multiple enlistments of the same remote repository share objects on the + same device. + +### Advanced Options + +The options below are not intended for use by a typical user. These are +usually used by build machines to create a temporary enlistment that +operates on a single commit. + +* `--single-branch`: Use this option to only download metadata for the branch + that will be checked out. This is helpful for build machines that target + a remote with many branches. Any `git fetch` commands after the clone will + still ask for all branches. + +Removing a Scalar Clone +----------------------- + +Since the `scalar clone` command sets up a file-system watcher (when available), +that watcher could prevent deleting the enlistment. Run `scalar delete ` +from outside of your enlistment to unregister the enlistment from the filesystem +watcher and delete the enlistment at ``. diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md new file mode 100644 index 0000000000..4f56e2b0eb --- /dev/null +++ b/contrib/scalar/docs/index.md @@ -0,0 +1,54 @@ +Scalar: Enabling Git at Scale +============================= + +Scalar is a tool that helps Git scale to some of the largest Git repositories. +It achieves this by enabling some advanced Git features, such as: + +* *Partial clone:* reduces time to get a working repository by not + downloading all Git objects right away. + +* *Background prefetch:* downloads Git object data from all remotes every + hour, reducing the amount of time for foreground `git fetch` calls. + +* *Sparse-checkout:* limits the size of your working directory. + +* *File system monitor:* tracks the recently modified files and eliminates + the need for Git to scan the entire worktree. + +* *Commit-graph:* accelerates commit walks and reachability calculations, + speeding up commands like `git log`. + +* *Multi-pack-index:* enables fast object lookups across many pack-files. + +* *Incremental repack:* Repacks the packed Git data into fewer pack-file + without disrupting concurrent commands by using the multi-pack-index. + +By running `scalar register` in any Git repo, Scalar will automatically enable +these features for that repo (except partial clone) and start running suggested +maintenance in the background using +[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance). + +Repos cloned with the `scalar clone` command use partial clone or the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md) +to significantly reduce the amount of data required to get started +using a repository. By delaying all blob downloads until they are required, +Scalar allows you to work with very large repositories quickly. The GVFS +protocol allows a network of _cache servers_ to serve objects with lower +latency and higher throughput. The cache servers also reduce load on the +central server. + +Documentation +------------- + +* [Getting Started](getting-started.md): Get started with Scalar. + Includes `scalar register`, `scalar unregister`, `scalar clone`, and + `scalar delete`. + +* [Troubleshooting](troubleshooting.md): + Collect diagnostic information or update custom settings. Includes + `scalar diagnose` and `scalar cache-server`. + +* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way + it does, and how do we make decisions about its future? + +* [Frequently Asked Questions](faq.md) diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md new file mode 100644 index 0000000000..e3dfa025a2 --- /dev/null +++ b/contrib/scalar/docs/philosophy.md @@ -0,0 +1,71 @@ +The Philosophy of Scalar +======================== + +The team building Scalar has **opinions** about Git performance. Scalar +takes out the guesswork by automatically configuring your Git repositories +to take advantage of the latest and greatest features. It is difficult to +say that these are the absolute best settings for every repository, but +these settings do work for some of the largest repositories in the world. + +Scalar intends to do very little more than the standard Git client. We +actively implement new features into Git instead of Scalar, then update +Scalar only to configure those new settings. In particular, we ported +features like background maintenance to Git to make Scalar simpler and +make Git more powerful. + +Scalar ships inside [a custom version of Git][microsoft-git], but we are +working to make it available in other forks of Git. The only feature +that is not intended to ever reach the standard Git client is Scalar's use +of [the GVFS Protocol][gvfs-protocol], which is essentially an older +version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/) +that was available first in Azure Repos. Services such as GitHub support +only partial clone instead of the GVFS protocol because that is the +standard adopted by the Git project. If your hosting service supports +partial clone, then we absolutely recommend it as a way to greatly speed +up your clone and fetch times and to reduce how much disk space your Git +repository requires. Scalar will help with this! + +If you don't use the GVFS Protocol, then most of the value of Scalar can +be found in the core Git client. However, most of the advanced features +that really optimize Git's performance are off by default for compatibility +reasons. To really take advantage of Git's latest and greatest features, +you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config) +and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). +Even if you do all that work and customize your Git settings on your machines, +you likely will want to share those settings with other team members. +Or, you can just use Scalar! + +Using `scalar register` on an existing Git repository will give you these +benefits: + +* Additional compression of your `.git/index` file. +* Hourly background `git fetch` operations, keeping you in-sync with your + remotes. +* Advanced data structures, such as the `commit-graph` and `multi-pack-index` + are updated automatically in the background. +* If using macOS or Windows, then Scalar configures Git's builtin File System + Monitor, providing faster commands such as `git status` or `git add`. + +Additionally, if you use `scalar clone` to create a new repository, then +you will automatically get these benefits: + +* Use Git's partial clone feature to only download the files you need for + your current checkout. +* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the + number of files required in your working directory. + [Read more about sparse-checkout here.][sparse-checkout-blog] +* Create the Git repository inside `/src` to make it easy to + place build artifacts outside of the Git repository, such as in + `/bin` or `/packages`. + +We also admit that these **opinions** can always be improved! If you have +an idea of how to improve our setup, consider +[creating an issue](https://github.com/microsoft/scalar/issues/new) or +contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382) +[issues](https://github.com/microsoft/scalar/issues/388) have already +improved our configuration settings and roadmap! + +[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md +[microsoft-git]: https://github.com/microsoft/git +[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout +[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/ diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md new file mode 100644 index 0000000000..c54d2438f2 --- /dev/null +++ b/contrib/scalar/docs/troubleshooting.md @@ -0,0 +1,40 @@ +Troubleshooting +=============== + +Diagnosing Issues +----------------- + +The `scalar diagnose` command collects logs and config details for the current +repository. The resulting zip file helps root-cause issues. + +When run inside your repository, creates a zip file containing several important +files for that repository. This includes: + +* Configuration files from your `.git` folder, such as the `config` file, + `index`, `hooks`, and `refs`. + +* A summary of your Git object database, including the number of loose objects + and the names and sizes of pack-files. + +As the `diagnose` command completes, it provides the path of the resulting +zip file. This zip can be attached to bug reports to make the analysis easier. + +Modifying Configuration Values +------------------------------ + +The Scalar-specific configuration is only available for repos using the +GVFS protocol. + +### Cache Server URL + +When using an enlistment cloned with `scalar clone` and the GVFS protocol, +you will have a value called the cache server URL. Cache servers are a feature +of the GVFS protocol to provide low-latency access to the on-demand object +requests. This modifies the `gvfs.cache-server` setting in your local Git config +file. + +Run `scalar cache-server --get` to see the current cache server. + +Run `scalar cache-server --list` to see the available cache server URLs. + +Run `scalar cache-server --set=` to set your cache server to ``. diff --git a/diagnose.c b/diagnose.c index 8f26569896..09515e39bb 100644 --- a/diagnose.c +++ b/diagnose.c @@ -7,6 +7,7 @@ #include "strvec.h" #include "object-store.h" #include "packfile.h" +#include "config.h" struct archive_dir { const char *path; @@ -102,6 +103,39 @@ cleanup: return dtype; } +static void dir_stats(struct strbuf *buf, const char *path) +{ + DIR *dir = opendir(path); + struct dirent *e; + struct stat e_stat; + struct strbuf file_path = STRBUF_INIT; + size_t base_path_len; + + if (!dir) + return; + + strbuf_addstr(buf, "Contents of "); + strbuf_add_absolute_path(buf, path); + strbuf_addstr(buf, ":\n"); + + strbuf_add_absolute_path(&file_path, path); + strbuf_addch(&file_path, '/'); + base_path_len = file_path.len; + + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) { + strbuf_setlen(&file_path, base_path_len); + strbuf_addstr(&file_path, e->d_name); + if (!stat(file_path.buf, &e_stat)) + strbuf_addf(buf, "%-70s %16"PRIuMAX"\n", + e->d_name, + (uintmax_t)e_stat.st_size); + } + + strbuf_release(&file_path); + closedir(dir); +} + static int count_files(struct strbuf *path) { DIR *dir = opendir(path->buf); @@ -214,7 +248,8 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) struct strvec archiver_args = STRVEC_INIT; char **argv_copy = NULL; int stdout_fd = -1, archiver_fd = -1; - struct strbuf buf = STRBUF_INIT; + char *cache_server_url = NULL, *shared_cache = NULL; + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; int res, i; struct archive_dir archive_dirs[] = { { ".git", 0 }, @@ -249,6 +284,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) get_version_info(&buf, 1); strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree); + + git_config_get_string("gvfs.cache-server", &cache_server_url); + git_config_get_string("gvfs.sharedCache", &shared_cache); + strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n", + cache_server_url ? cache_server_url : "None", + shared_cache ? shared_cache : "None"); + get_disk_info(&buf); write_or_die(stdout_fd, buf.buf, buf.len); strvec_pushf(&archiver_args, @@ -279,6 +321,52 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) } } + if (shared_cache) { + size_t path_len; + + strbuf_reset(&buf); + strbuf_addf(&path, "%s/pack", shared_cache); + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:"); + dir_stats(&buf, path.buf); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:"); + loose_objs_stats(&buf, shared_cache); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info", shared_cache); + path_len = path.len; + + if (is_directory(path.buf)) { + DIR *dir = opendir(path.buf); + struct dirent *e; + + while ((e = readdir(dir))) { + if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name)) + continue; + if (e->d_type == DT_DIR) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name); + + strbuf_setlen(&path, path_len); + strbuf_addch(&path, '/'); + strbuf_addstr(&path, e->d_name); + + if (strbuf_read_file(&buf, path.buf, 0) < 0) { + res = error_errno(_("could not read '%s'"), path.buf); + goto diagnose_cleanup; + } + strvec_push(&archiver_args, buf.buf); + } + closedir(dir); + } + } + strvec_pushl(&archiver_args, "--prefix=", oid_to_hex(the_hash_algo->empty_tree), "--", NULL); @@ -292,10 +380,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) goto diagnose_cleanup; } - fprintf(stderr, "\n" - "Diagnostics complete.\n" - "All of the gathered info is captured in '%s'\n", - zip_path->buf); + strbuf_reset(&buf); + strbuf_addf(&buf, "\n" + "Diagnostics complete.\n" + "All of the gathered info is captured in '%s'\n", + zip_path->buf); + write_or_die(stdout_fd, buf.buf, buf.len); + write_or_die(2, buf.buf, buf.len); diagnose_cleanup: if (archiver_fd >= 0) { @@ -306,6 +397,8 @@ diagnose_cleanup: free(argv_copy); strvec_clear(&archiver_args); strbuf_release(&buf); + free(cache_server_url); + free(shared_cache); return res; } diff --git a/dir.c b/dir.c index 01329773c1..c89f17618c 100644 --- a/dir.c +++ b/dir.c @@ -3127,6 +3127,8 @@ static int cmp_icase(char a, char b) { if (a == b) return 0; + if (is_dir_sep(a)) + return is_dir_sep(b) ? 0 : -1; if (ignore_case) return toupper(a) - toupper(b); return a - b; diff --git a/gvfs-helper.c b/gvfs-helper.c index ce89edc33c..e8e6fbbdd9 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -202,6 +202,12 @@ // [2] Documentation/technical/long-running-process-protocol.txt // [3] See GIT_TRACE_PACKET // +// endpoint +// +// Fetch the given endpoint from the main Git server (specifying +// `gvfs/config` as endpoint is idempotent to the `config` +// command mentioned above). +// ////////////////////////////////////////////////////////////////// #include "cache.h" @@ -3110,18 +3116,20 @@ static void do_req__with_fallback(const char *url_component, * * Return server's response buffer. This is probably a raw JSON string. */ -static void do__http_get__gvfs_config(struct gh__response_status *status, - struct strbuf *config_data) +static void do__http_get__simple_endpoint(struct gh__response_status *status, + struct strbuf *response, + const char *endpoint, + const char *tr2_label) { struct gh__request_params params = GH__REQUEST_PARAMS_INIT; - strbuf_addstr(¶ms.tr2_label, "GET/config"); + strbuf_addstr(¶ms.tr2_label, tr2_label); params.b_is_post = 0; params.b_write_to_file = 0; /* cache-servers do not handle gvfs/config REST calls */ params.b_permit_cache_server_if_defined = 0; - params.buffer = config_data; + params.buffer = response; params.objects_mode = GH__OBJECTS_MODE__NONE; params.object_count = 1; /* a bit of a lie */ @@ -3143,15 +3151,22 @@ static void do__http_get__gvfs_config(struct gh__response_status *status, * see any need to report progress on the upload side of * the GET. So just report progress on the download side. */ - strbuf_addstr(¶ms.progress_base_phase3_msg, - "Receiving gvfs/config"); + strbuf_addf(¶ms.progress_base_phase3_msg, + "Receiving %s", endpoint); } - do_req__with_fallback("gvfs/config", ¶ms, status); + do_req__with_fallback(endpoint, ¶ms, status); gh__request_params__release(¶ms); } +static void do__http_get__gvfs_config(struct gh__response_status *status, + struct strbuf *config_data) +{ + do__http_get__simple_endpoint(status, config_data, "gvfs/config", + "GET/config"); +} + static void setup_gvfs_objects_progress(struct gh__request_params *params, unsigned long num, unsigned long den) { @@ -3596,6 +3611,35 @@ static enum gh__error_code do_sub_cmd__config(int argc, const char **argv) return ec; } +static enum gh__error_code do_sub_cmd__endpoint(int argc, const char **argv) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct strbuf data = STRBUF_INIT; + enum gh__error_code ec = GH__ERROR_CODE__OK; + const char *endpoint; + + if (argc != 2) + return GH__ERROR_CODE__ERROR; + endpoint = argv[1]; + + trace2_cmd_mode(endpoint); + + finish_init(0); + + do__http_get__simple_endpoint(&status, &data, endpoint, endpoint); + ec = status.ec; + + if (ec == GH__ERROR_CODE__OK) + printf("%s\n", data.buf); + else + error("config: %s", status.error_message.buf); + + gh__response_status__release(&status); + strbuf_release(&data); + + return ec; +} + /* * Read a list of objects from stdin and fetch them as a series of * single object HTTP GET requests. @@ -4087,6 +4131,9 @@ static enum gh__error_code do_sub_cmd(int argc, const char **argv) if (!strcmp(argv[0], "config")) return do_sub_cmd__config(argc, argv); + if (!strcmp(argv[0], "endpoint")) + return do_sub_cmd__endpoint(argc, argv); + if (!strcmp(argv[0], "prefetch")) return do_sub_cmd__prefetch(argc, argv); diff --git a/json-parser.c b/json-parser.c new file mode 100644 index 0000000000..30799e17dc --- /dev/null +++ b/json-parser.c @@ -0,0 +1,182 @@ +#include "cache.h" +#include "json-parser.h" + +static int reset_iterator(struct json_iterator *it) +{ + it->p = it->begin = it->json; + strbuf_release(&it->key); + strbuf_release(&it->string_value); + it->type = JSON_NULL; + return -1; +} + +static int parse_json_string(struct json_iterator *it, struct strbuf *out) +{ + const char *begin = it->p; + + if (*(it->p)++ != '"') + return error("expected double quote: '%.*s'", 5, begin), + reset_iterator(it); + + strbuf_reset(&it->string_value); +#define APPEND(c) strbuf_addch(out, c) + while (*it->p != '"') { + switch (*it->p) { + case '\0': + return error("incomplete string: '%s'", begin), + reset_iterator(it); + case '\\': + it->p++; + if (*it->p == '\\' || *it->p == '"') + APPEND(*it->p); + else if (*it->p == 'b') + APPEND(8); + else if (*it->p == 't') + APPEND(9); + else if (*it->p == 'n') + APPEND(10); + else if (*it->p == 'f') + APPEND(12); + else if (*it->p == 'r') + APPEND(13); + else if (*it->p == 'u') { + unsigned char binary[2]; + int i; + + if (hex_to_bytes(binary, it->p + 1, 2) < 0) + return error("invalid: '%.*s'", + 6, it->p - 1), + reset_iterator(it); + it->p += 4; + + i = (binary[0] << 8) | binary[1]; + if (i < 0x80) + APPEND(i); + else if (i < 0x0800) { + APPEND(0xc0 | ((i >> 6) & 0x1f)); + APPEND(0x80 | (i & 0x3f)); + } else if (i < 0x10000) { + APPEND(0xe0 | ((i >> 12) & 0x0f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } else { + APPEND(0xf0 | ((i >> 18) & 0x07)); + APPEND(0x80 | ((i >> 12) & 0x3f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } + } + break; + default: + APPEND(*it->p); + } + it->p++; + } + + it->end = it->p++; + return 0; +} + +static void skip_whitespace(struct json_iterator *it) +{ + while (isspace(*it->p)) + it->p++; +} + +int iterate_json(struct json_iterator *it) +{ + skip_whitespace(it); + it->begin = it->p; + + switch (*it->p) { + case '\0': + return reset_iterator(it), 0; + case 'n': + if (!starts_with(it->p, "null")) + return error("unexpected value: %.*s", 4, it->p), + reset_iterator(it); + it->type = JSON_NULL; + it->end = it->p = it->begin + 4; + break; + case 't': + if (!starts_with(it->p, "true")) + return error("unexpected value: %.*s", 4, it->p), + reset_iterator(it); + it->type = JSON_TRUE; + it->end = it->p = it->begin + 4; + break; + case 'f': + if (!starts_with(it->p, "false")) + return error("unexpected value: %.*s", 5, it->p), + reset_iterator(it); + it->type = JSON_FALSE; + it->end = it->p = it->begin + 5; + break; + case '-': case '.': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + it->type = JSON_NUMBER; + it->end = it->p = it->begin + strspn(it->p, "-.0123456789"); + break; + case '"': + it->type = JSON_STRING; + if (parse_json_string(it, &it->string_value) < 0) + return -1; + break; + case '[': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int i = 0, res; + + for (it->p++, skip_whitespace(it); *it->p != ']'; i++) { + strbuf_addf(&it->key, "[%d]", i); + + if ((res = iterate_json(it))) + return reset_iterator(it), res; + strbuf_setlen(&it->key, key_offset); + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + + it->type = JSON_ARRAY; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + case '{': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int res; + + strbuf_addch(&it->key, '.'); + for (it->p++, skip_whitespace(it); *it->p != '}'; ) { + strbuf_setlen(&it->key, key_offset + 1); + if (parse_json_string(it, &it->key) < 0) + return -1; + skip_whitespace(it); + if (*(it->p)++ != ':') + return error("expected colon: %.*s", 5, it->p), + reset_iterator(it); + + if ((res = iterate_json(it))) + return res; + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + strbuf_setlen(&it->key, key_offset); + + it->type = JSON_OBJECT; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + } + + return it->fn(it); +} diff --git a/json-parser.h b/json-parser.h new file mode 100644 index 0000000000..ce1fdc5ee2 --- /dev/null +++ b/json-parser.h @@ -0,0 +1,29 @@ +#ifndef JSON_PARSER_H +#define JSON_PARSER_H + +#include "strbuf.h" + +struct json_iterator { + const char *json, *p, *begin, *end; + struct strbuf key, string_value; + enum { + JSON_NULL = 0, + JSON_FALSE, + JSON_TRUE, + JSON_NUMBER, + JSON_STRING, + JSON_ARRAY, + JSON_OBJECT + } type; + int (*fn)(struct json_iterator *it); + void *fn_data; +}; +#define JSON_ITERATOR_INIT(json_, fn_, fn_data_) { \ + .json = json_, .p = json_, \ + .key = STRBUF_INIT, .string_value = STRBUF_INIT, \ + .fn = fn_, .fn_data = fn_data_ \ +} + +int iterate_json(struct json_iterator *it); + +#endif diff --git a/path.h b/path.h index 0a59c85a62..db417a43bd 100644 --- a/path.h +++ b/path.h @@ -179,6 +179,10 @@ const char *git_path_auto_merge(struct repository *r); const char *git_path_fetch_head(struct repository *r); const char *git_path_shallow(struct repository *r); +/** + * Remove the last path component from 'path' except if 'path' is root. + */ +void strip_last_path_component(struct strbuf *path); int ends_with_path_components(const char *path, const char *components); diff --git a/scalar.c b/scalar.c index ca19b95ce4..0b0d6297da 100644 --- a/scalar.c +++ b/scalar.c @@ -14,6 +14,13 @@ #include "dir.h" #include "packfile.h" #include "help.h" +#include "json-parser.h" +#include "remote.h" +#include "path.h" + +static int is_unattended(void) { + return git_env_bool("Scalar_UNATTENDED", 0); +} static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, @@ -41,6 +48,9 @@ static void setup_enlistment_directory(int argc, const char **argv, die(_("need a working directory")); strbuf_trim_trailing_dir_sep(&path); +#ifdef GIT_WINDOWS_NATIVE + convert_slashes(path.buf); +#endif /* check if currently in enlistment root with src/ workdir */ len = path.len; @@ -67,20 +77,46 @@ static void setup_enlistment_directory(int argc, const char **argv, strbuf_release(&path); } +static int git_retries = 3; + static int run_git(const char *arg, ...) { - struct child_process cmd = CHILD_PROCESS_INIT; va_list args; const char *p; + struct strvec argv = STRVEC_INIT; + int res = 0, attempts; va_start(args, arg); - strvec_push(&cmd.args, arg); + strvec_push(&argv, arg); while ((p = va_arg(args, const char *))) - strvec_push(&cmd.args, p); + strvec_push(&argv, p); va_end(args); - cmd.git_cmd = 1; - return run_command(&cmd); + for (attempts = 0, res = 1; + res && attempts < git_retries; + attempts++) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushv(&cmd.args, argv.v); + res = run_command(&cmd); + } + + strvec_clear(&argv); + return res; +} + +static const char *ensure_absolute_path(const char *path, char **absolute) +{ + struct strbuf buf = STRBUF_INIT; + + if (is_absolute_path(path)) + return path; + + strbuf_realpath_forgiving(&buf, path, 1); + free(*absolute); + *absolute = strbuf_detach(&buf, NULL); + return *absolute; } struct scalar_config { @@ -121,23 +157,7 @@ static int set_recommended_config(int reconfigure) { "core.FSCache", "true", 1 }, { "core.multiPackIndex", "true", 1 }, { "core.preloadIndex", "true", 1 }, -#ifndef WIN32 { "core.untrackedCache", "true", 1 }, -#else - /* - * Unfortunately, Scalar's Functional Tests demonstrated - * that the untracked cache feature is unreliable on Windows - * (which is a bummer because that platform would benefit the - * most from it). For some reason, freshly created files seem - * not to update the directory's `lastModified` time - * immediately, but the untracked cache would need to rely on - * that. - * - * Therefore, with a sad heart, we disable this very useful - * feature on Windows. - */ - { "core.untrackedCache", "false", 1 }, -#endif { "core.logAllRefUpdates", "true", 1 }, { "credential.https://dev.azure.com.useHttpPath", "true", 1 }, { "credential.validate", "false", 1 }, /* GCM4W-only */ @@ -164,11 +184,29 @@ static int set_recommended_config(int reconfigure) { "core.autoCRLF", "false" }, { "core.safeCRLF", "false" }, { "fetch.showForcedUpdates", "false" }, + { "core.configWriteLockTimeoutMS", "150" }, { NULL, NULL }, }; int i; char *value; + /* + * If a user has "core.usebuiltinfsmonitor" enabled, try to switch to + * the new (non-deprecated) setting (core.fsmonitor). + */ + if (!git_config_get_string("core.usebuiltinfsmonitor", &value)) { + char *dummy = NULL; + if (git_config_get_string("core.fsmonitor", &dummy) && + git_config_set_gently("core.fsmonitor", value) < 0) + return error(_("could not configure %s=%s"), + "core.fsmonitor", value); + if (git_config_set_gently("core.usebuiltinfsmonitor", NULL) < 0) + return error(_("could not configure %s=%s"), + "core.useBuiltinFSMonitor", "NULL"); + free(value); + free(dummy); + } + for (i = 0; config[i].key; i++) { if (set_scalar_config(config + i, reconfigure)) return error(_("could not configure %s=%s"), @@ -205,6 +243,11 @@ static int set_recommended_config(int reconfigure) static int toggle_maintenance(int enable) { + unsigned long ul; + + if (git_config_get_ulong("core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + return run_git("maintenance", enable ? "start" : "unregister", enable ? NULL : "--force", @@ -214,10 +257,14 @@ static int toggle_maintenance(int enable) static int add_or_remove_enlistment(int add) { int res; + unsigned long ul; if (!the_repository->worktree) die(_("Scalar enlistments require a worktree")); + if (git_config_get_ulong("core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + res = run_git("config", "--global", "--get", "--fixed-value", "scalar.repo", the_repository->worktree, NULL); @@ -305,6 +352,215 @@ static int set_config(const char *fmt, ...) return res; } +static int list_cache_server_urls(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_STRING && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].Url")) + printf("#%ld: %s\n", l, it->string_value.buf); + + return 0; +} + +/* Find N for which .CacheServers[N].GlobalDefault == true */ +static int get_cache_server_index(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_TRUE && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].GlobalDefault")) { + *(long *)it->fn_data = l; + return 1; + } + + return 0; +} + +struct cache_server_url_data { + char *key, *url; +}; + +/* Get .CacheServers[N].Url */ +static int get_cache_server_url(struct json_iterator *it) +{ + struct cache_server_url_data *data = it->fn_data; + + if (it->type == JSON_STRING && + !strcasecmp(data->key, it->key.buf)) { + data->url = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +static int can_url_support_gvfs(const char *url) +{ + return starts_with(url, "https://") || + (git_env_bool("GIT_TEST_ALLOW_GVFS_VIA_HTTP", 0) && + starts_with(url, "http://")); +} + +/* + * If `cache_server_url` is `NULL`, print the list to `stdout`. + * + * Since `gvfs-helper` requires a Git directory, this _must_ be run in + * a worktree. + */ +static int supports_gvfs_protocol(const char *url, char **cache_server_url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (!can_url_support_gvfs(url)) + return 0; + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, "config", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + long l = 0; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_cache_server_index, &l); + struct cache_server_url_data data = { .url = NULL }; + + if (!cache_server_url) { + it.fn = list_cache_server_urls; + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + strbuf_release(&out); + return 0; + } + + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + data.key = xstrfmt(".CacheServers[%ld].Url", l); + it.fn = get_cache_server_url; + it.fn_data = &data; + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + *cache_server_url = data.url; + free(data.key); + return 1; + } + strbuf_release(&out); + /* error out quietly, unless we wanted to list URLs */ + return cache_server_url ? + 0 : error(_("Could not access gvfs/config endpoint")); +} + +static char *default_cache_root(const char *root) +{ + const char *env; + + if (is_unattended()) { + struct strbuf path = STRBUF_INIT; + strbuf_addstr(&path, root); + strip_last_path_component(&path); + strbuf_addstr(&path, "/.scalarCache"); + return strbuf_detach(&path, NULL); + } + +#ifdef WIN32 + (void)env; + return xstrfmt("%.*s.scalarCache", offset_1st_component(root), root); +#elif defined(__APPLE__) + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.scalarCache", env); + return NULL; +#else + if ((env = getenv("XDG_CACHE_HOME")) && *env) + return xstrfmt("%s/scalar", env); + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.cache/scalar", env); + return NULL; +#endif +} + +static int get_repository_id(struct json_iterator *it) +{ + if (it->type == JSON_STRING && + !strcasecmp(".repository.id", it->key.buf)) { + *(char **)it->fn_data = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +/* Needs to run this in a worktree; gvfs-helper requires a Git repository */ +static char *get_cache_key(const char *url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + char *cache_key = NULL; + + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (!git_env_bool("SCALAR_TEST_SKIP_VSTS_INFO", 0) && + can_url_support_gvfs(url)) { + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, + "endpoint", "vsts/info", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + char *id = NULL; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_repository_id, + &id); + + if (iterate_json(&it) < 0) + warning("JSON parse error (%s)", out.buf); + else if (id) + cache_key = xstrfmt("id_%s", id); + free(id); + } + } + + if (!cache_key) { + struct strbuf downcased = STRBUF_INIT; + int hash_algo_index = hash_algo_by_name("sha1"); + const struct git_hash_algo *hash_algo = hash_algo_index < 0 ? + the_hash_algo : &hash_algos[hash_algo_index]; + git_hash_ctx ctx; + unsigned char hash[GIT_MAX_RAWSZ]; + + strbuf_addstr(&downcased, url); + strbuf_tolower(&downcased); + + hash_algo->init_fn(&ctx); + hash_algo->update_fn(&ctx, downcased.buf, downcased.len); + hash_algo->final_fn(hash, &ctx); + + strbuf_release(&downcased); + + cache_key = xstrfmt("url_%s", + hash_to_hex_algop(hash, hash_algo)); + } + + strbuf_release(&out); + return cache_key; +} + static char *remote_default_branch(const char *url) { struct child_process cp = CHILD_PROCESS_INIT; @@ -402,10 +658,51 @@ void load_builtin_commands(const char *prefix, struct cmdnames *cmds) die("not implemented"); } +static int init_shared_object_cache(const char *url, + const char *local_cache_root) +{ + struct strbuf buf = STRBUF_INIT; + int res = 0; + char *cache_key = NULL, *shared_cache_path = NULL; + + if (!(cache_key = get_cache_key(url))) { + res = error(_("could not determine cache key for '%s'"), url); + goto cleanup; + } + + shared_cache_path = xstrfmt("%s/%s", local_cache_root, cache_key); + if (set_config("gvfs.sharedCache=%s", shared_cache_path)) { + res = error(_("could not configure shared cache")); + goto cleanup; + } + + strbuf_addf(&buf, "%s/pack", shared_cache_path); + switch (safe_create_leading_directories(buf.buf)) { + case SCLD_OK: case SCLD_EXISTS: + break; /* okay */ + default: + res = error_errno(_("could not initialize '%s'"), buf.buf); + goto cleanup; + } + + write_file(git_path("objects/info/alternates"),"%s\n", shared_cache_path); + + cleanup: + strbuf_release(&buf); + free(shared_cache_path); + free(cache_key); + return res; +} + static int cmd_clone(int argc, const char **argv) { + int dummy = 0; const char *branch = NULL; int full_clone = 0, single_branch = 0, show_progress = isatty(2); + const char *cache_server_url = NULL, *local_cache_root = NULL; + char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; + const char *enlistment_parent; + int no_src = 0; struct option clone_options[] = { OPT_STRING('b', "branch", &branch, N_(""), N_("branch to checkout after clone")), @@ -414,6 +711,16 @@ static int cmd_clone(int argc, const char **argv) OPT_BOOL(0, "single-branch", &single_branch, N_("only download metadata for the branch that will " "be checked out")), + OPT_BOOL(0, "no-src", &no_src, + N_("skip creating a 'src' directory")), + OPT_STRING(0, "cache-server-url", &cache_server_url, + N_(""), + N_("the url or friendly name of the cache server")), + OPT_STRING(0, "local-cache-path", &local_cache_root, + N_(""), + N_("override the path for the local Scalar cache")), + OPT_HIDDEN_BOOL(0, "no-fetch-commits-and-trees", + &dummy, N_("no longer used")), OPT_END(), }; const char * const clone_usage[] = { @@ -424,6 +731,7 @@ static int cmd_clone(int argc, const char **argv) char *enlistment = NULL, *dir = NULL; struct strbuf buf = STRBUF_INIT; int res; + int gvfs_protocol; argc = parse_options(argc, argv, NULL, clone_options, clone_usage, 0); @@ -453,7 +761,25 @@ static int cmd_clone(int argc, const char **argv) if (is_directory(enlistment)) die(_("directory '%s' exists already"), enlistment); - dir = xstrfmt("%s/src", enlistment); + ensure_absolute_path(enlistment, &enlistment); + + if (!no_src) { + dir = xstrfmt("%s/src", enlistment); + enlistment_parent = "../.."; + } else { + dir = xstrdup(enlistment); + enlistment_parent = ".."; + } + + if (!local_cache_root) + local_cache_root = local_cache_root_abs = + default_cache_root(enlistment); + else + local_cache_root = ensure_absolute_path(local_cache_root, + &local_cache_root_abs); + + if (!local_cache_root) + die(_("could not determine local cache root")); strbuf_reset(&buf); if (branch) @@ -474,8 +800,28 @@ static int cmd_clone(int argc, const char **argv) setup_git_directory(); + git_config(git_default_config, NULL); + + /* + * This `dir_inside_of()` call relies on git_config() having parsed the + * newly-initialized repository config's `core.ignoreCase` value. + */ + if (dir_inside_of(local_cache_root, dir) >= 0) { + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, enlistment); + if (chdir(enlistment_parent) < 0 || + remove_dir_recursively(&path, 0) < 0) + die(_("'--local-cache-path' cannot be inside the src " + "folder;\nCould not remove '%s'"), enlistment); + + die(_("'--local-cache-path' cannot be inside the src folder")); + } + /* common-main already logs `argv` */ trace2_def_repo(the_repository); + trace2_data_intmax("scalar", the_repository, "unattended", + is_unattended()); if (!branch && !(branch = remote_default_branch(url))) { res = error(_("failed to get default branch for '%s'"), url); @@ -486,13 +832,48 @@ static int cmd_clone(int argc, const char **argv) set_config("remote.origin.fetch=" "+refs/heads/%s:refs/remotes/origin/%s", single_branch ? branch : "*", - single_branch ? branch : "*") || - set_config("remote.origin.promisor=true") || - set_config("remote.origin.partialCloneFilter=blob:none")) { + single_branch ? branch : "*")) { res = error(_("could not configure remote in '%s'"), dir); goto cleanup; } + if (set_config("credential.https://dev.azure.com.useHttpPath=true")) { + res = error(_("could not configure credential.useHttpPath")); + goto cleanup; + } + + gvfs_protocol = cache_server_url || + supports_gvfs_protocol(url, &default_cache_server_url); + + if (gvfs_protocol) { + if ((res = init_shared_object_cache(url, local_cache_root))) + goto cleanup; + if (!cache_server_url) + cache_server_url = default_cache_server_url; + if (set_config("core.useGVFSHelper=true") || + set_config("core.gvfs=150") || + set_config("http.version=HTTP/1.1")) { + res = error(_("could not turn on GVFS helper")); + goto cleanup; + } + if (cache_server_url && + set_config("gvfs.cache-server=%s", cache_server_url)) { + res = error(_("could not configure cache server")); + goto cleanup; + } + if (cache_server_url) + fprintf(stderr, "Cache server URL: %s\n", + cache_server_url); + } else { + if (set_config("core.useGVFSHelper=false") || + set_config("remote.origin.promisor=true") || + set_config("remote.origin.partialCloneFilter=blob:none")) { + res = error(_("could not configure partial clone in " + "'%s'"), dir); + goto cleanup; + } + } + if (!full_clone && (res = run_git("sparse-checkout", "init", "--cone", NULL))) goto cleanup; @@ -503,6 +884,11 @@ static int cmd_clone(int argc, const char **argv) if ((res = run_git("fetch", "--quiet", show_progress ? "--progress" : "--no-progress", "origin", NULL))) { + if (gvfs_protocol) { + res = error(_("failed to prefetch commits and trees")); + goto cleanup; + } + warning(_("partial clone failed; attempting full clone")); if (set_config("remote.origin.promisor") || @@ -535,6 +921,8 @@ cleanup: free(enlistment); free(dir); strbuf_release(&buf); + free(default_cache_server_url); + free(local_cache_root_abs); return res; } @@ -556,6 +944,8 @@ static int cmd_diagnose(int argc, const char **argv) setup_enlistment_directory(argc, argv, usage, options, &diagnostics_root); strbuf_addstr(&diagnostics_root, "/.scalarDiagnostics"); + /* Here, a failure should not repeat itself. */ + git_retries = 1; res = run_git("diagnose", "--mode=all", "-s", "%Y%m%d_%H%M%S", "-o", diagnostics_root.buf, NULL); @@ -652,6 +1042,7 @@ static int cmd_reconfigure(int argc, const char **argv) git_config(get_scalar_repos, &scalar_repos); for (i = 0; i < scalar_repos.nr; i++) { + int failed = 0; const char *dir = scalar_repos.items[i].string; strbuf_reset(&commondir); @@ -662,30 +1053,51 @@ static int cmd_reconfigure(int argc, const char **argv) if (errno != ENOENT) { warning_errno(_("could not switch to '%s'"), dir); - res = -1; - continue; + failed = -1; + goto loop_end; } strbuf_addstr(&buf, dir); if (remove_deleted_enlistment(&buf)) - res = error(_("could not remove stale " - "scalar.repo '%s'"), dir); + failed = error(_("could not remove stale " + "scalar.repo '%s'"), dir); else - warning(_("removing stale scalar.repo '%s'"), + warning(_("removed stale scalar.repo '%s'"), dir); strbuf_release(&buf); - } else if (discover_git_directory(&commondir, &gitdir) < 0) { - warning_errno(_("git repository gone in '%s'"), dir); - res = -1; - } else { - git_config_clear(); + goto loop_end; + } - the_repository = &r; - r.commondir = commondir.buf; - r.gitdir = gitdir.buf; + switch (discover_git_directory_reason(&commondir, &gitdir)) { + case GIT_DIR_INVALID_OWNERSHIP: + warning(_("repository at '%s' has different owner"), dir); + failed = -1; + goto loop_end; - if (set_recommended_config(1) < 0) - res = -1; + case GIT_DIR_DISCOVERED: + break; + + default: + warning(_("repository not found in '%s'"), dir); + failed = -1; + break; + } + + git_config_clear(); + + the_repository = &r; + r.commondir = commondir.buf; + r.gitdir = gitdir.buf; + + if (set_recommended_config(1) < 0) + failed = -1; + +loop_end: + if (failed) { + res = failed; + warning(_("to unregister this repository from Scalar, run\n" + "\tgit config --global --unset --fixed-value scalar.repo \"%s\""), + dir); } } @@ -885,6 +1297,77 @@ static int cmd_version(int argc, const char **argv) return 0; } +static int cmd_cache_server(int argc, const char **argv) +{ + int get = 0; + char *set = NULL, *list = NULL; + const char *default_remote = "(default)"; + struct option options[] = { + OPT_BOOL(0, "get", &get, + N_("get the configured cache-server URL")), + OPT_STRING(0, "set", &set, N_("URL"), + N_("configure the cache-server to use")), + { OPTION_STRING, 0, "list", &list, N_("remote"), + N_("list the possible cache-server URLs"), + PARSE_OPT_OPTARG, NULL, (intptr_t) default_remote }, + OPT_END(), + }; + const char * const usage[] = { + N_("scalar cache_server " + "[--get | --set | --list []] []"), + NULL + }; + int res = 0; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (get + !!set + !!list > 1) + usage_msg_opt(_("--get/--set/--list are mutually exclusive"), + usage, options); + + setup_enlistment_directory(argc, argv, usage, options, NULL); + + if (list) { + const char *name = list, *url = list; + + if (list == default_remote) + list = NULL; + + if (!list || !strchr(list, '/')) { + struct remote *remote; + + /* Look up remote */ + remote = remote_get(list); + if (!remote) { + error("no such remote: '%s'", name); + free(list); + return 1; + } + if (!remote->url) { + free(list); + return error(_("remote '%s' has no URLs"), + name); + } + url = remote->url[0]; + } + res = supports_gvfs_protocol(url, NULL); + free(list); + } else if (set) { + res = set_config("gvfs.cache-server=%s", set); + free(set); + } else { + char *url = NULL; + + printf("Using cache server: %s\n", + git_config_get_string("gvfs.cache-server", &url) ? + "(undefined)" : url); + free(url); + } + + return !!res; +} + static struct { const char *name; int (*fn)(int, const char **); @@ -899,6 +1382,7 @@ static struct { { "help", cmd_help }, { "version", cmd_version }, { "diagnose", cmd_diagnose }, + { "cache-server", cmd_cache_server }, { NULL, NULL}, }; @@ -907,6 +1391,12 @@ int cmd_main(int argc, const char **argv) struct strbuf scalar_usage = STRBUF_INIT; int i; + if (is_unattended()) { + setenv("GIT_ASKPASS", "", 0); + setenv("GIT_TERMINAL_PROMPT", "false", 0); + git_config_push_parameter("credential.interactive=never"); + } + while (argc > 1 && *argv[1] == '-') { if (!strcmp(argv[1], "-C")) { if (argc < 3) @@ -930,6 +1420,9 @@ int cmd_main(int argc, const char **argv) argv++; argc--; + if (!strcmp(argv[0], "config")) + argv[0] = "reconfigure"; + for (i = 0; builtins[i].name; i++) if (!strcmp(builtins[i].name, argv[0])) return !!builtins[i].fn(argc, argv); diff --git a/setup.c b/setup.c index 852b3d9141..ca3f83cc6d 100644 --- a/setup.c +++ b/setup.c @@ -1205,19 +1205,6 @@ static const char *allowed_bare_repo_to_string( return NULL; } -enum discovery_result { - GIT_DIR_NONE = 0, - GIT_DIR_EXPLICIT, - GIT_DIR_DISCOVERED, - GIT_DIR_BARE, - /* these are errors */ - GIT_DIR_HIT_CEILING = -1, - GIT_DIR_HIT_MOUNT_POINT = -2, - GIT_DIR_INVALID_GITFILE = -3, - GIT_DIR_INVALID_OWNERSHIP = -4, - GIT_DIR_DISALLOWED_BARE = -5, -}; - /* * We cannot decide in this function whether we are in the work tree or * not, since the config can only be read _after_ this function was called. @@ -1368,21 +1355,22 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, } } -int discover_git_directory(struct strbuf *commondir, - struct strbuf *gitdir) +enum discovery_result discover_git_directory_reason(struct strbuf *commondir, + struct strbuf *gitdir) { struct strbuf dir = STRBUF_INIT, err = STRBUF_INIT; size_t gitdir_offset = gitdir->len, cwd_len; size_t commondir_offset = commondir->len; struct repository_format candidate = REPOSITORY_FORMAT_INIT; + enum discovery_result result; if (strbuf_getcwd(&dir)) - return -1; + return GIT_DIR_CWD_FAILURE; cwd_len = dir.len; - if (setup_git_directory_gently_1(&dir, gitdir, NULL, 0) <= 0) { + if ((result = setup_git_directory_gently_1(&dir, gitdir, NULL, 0)) <= 0) { strbuf_release(&dir); - return -1; + return result; } /* @@ -1412,7 +1400,7 @@ int discover_git_directory(struct strbuf *commondir, strbuf_setlen(commondir, commondir_offset); strbuf_setlen(gitdir, gitdir_offset); clear_repository_format(&candidate); - return -1; + return GIT_DIR_INVALID_FORMAT; } /* take ownership of candidate.partial_clone */ @@ -1421,7 +1409,7 @@ int discover_git_directory(struct strbuf *commondir, candidate.partial_clone = NULL; clear_repository_format(&candidate); - return 0; + return result; } const char *setup_git_directory_gently(int *nongit_ok) @@ -1513,9 +1501,11 @@ const char *setup_git_directory_gently(int *nongit_ok) *nongit_ok = 1; break; case GIT_DIR_NONE: + case GIT_DIR_CWD_FAILURE: + case GIT_DIR_INVALID_FORMAT: /* * As a safeguard against setup_git_directory_gently_1 returning - * this value, fallthrough to BUG. Otherwise it is possible to + * these values, fallthrough to BUG. Otherwise it is possible to * set startup_info->have_repository to 1 when we did nothing to * find a repository. */ diff --git a/t/helper/test-gvfs-protocol.c b/t/helper/test-gvfs-protocol.c index 97b3d363bf..a437248a10 100644 --- a/t/helper/test-gvfs-protocol.c +++ b/t/helper/test-gvfs-protocol.c @@ -1481,6 +1481,8 @@ done: static enum worker_result dispatch(struct req *req) { + static regex_t *smart_http_regex; + static int initialized; const char *method; enum worker_result wr; @@ -1529,6 +1531,53 @@ static enum worker_result dispatch(struct req *req) return do__gvfs_prefetch__get(req); } + if (!initialized) { + smart_http_regex = xmalloc(sizeof(*smart_http_regex)); + if (regcomp(smart_http_regex, "^/(HEAD|info/refs|" + "objects/info/[^/]+|git-(upload|receive)-pack)$", + REG_EXTENDED)) { + warning("could not compile smart HTTP regex"); + smart_http_regex = NULL; + } + initialized = 1; + } + + if (smart_http_regex && + !regexec(smart_http_regex, req->uri_base.buf, 0, NULL, 0)) { + const char *ok = "HTTP/1.1 200 OK\r\n"; + struct child_process cp = CHILD_PROCESS_INIT; + int i, res; + + if (write(1, ok, strlen(ok)) < 0) + return error(_("could not send '%s'"), ok); + + strvec_pushf(&cp.env, "REQUEST_METHOD=%s", method); + strvec_pushf(&cp.env, "PATH_TRANSLATED=%s", + req->uri_base.buf); + /* Prevent MSYS2 from "converting to a Windows path" */ + strvec_pushf(&cp.env, + "MSYS2_ENV_CONV_EXCL=PATH_TRANSLATED"); + strvec_push(&cp.env, "SERVER_PROTOCOL=HTTP/1.1"); + if (req->quest_args.len) + strvec_pushf(&cp.env, "QUERY_STRING=%s", + req->quest_args.buf); + for (i = 0; i < req->header_list.nr; i++) { + const char *header = req->header_list.items[i].string; + if (!strncasecmp("Content-Type: ", header, 14)) + strvec_pushf(&cp.env, "CONTENT_TYPE=%s", + header + 14); + else if (!strncasecmp("Content-Length: ", header, 16)) + strvec_pushf(&cp.env, "CONTENT_LENGTH=%s", + header + 16); + } + cp.git_cmd = 1; + strvec_push(&cp.args, "http-backend"); + res = run_command(&cp); + close(1); + close(0); + return !!res; + } + return send_http_error(1, 501, "Not Implemented", -1, WR_OK | WR_HANGUP); } diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index 4432a30d10..43c4208cdc 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -7,6 +7,13 @@ test_description='test the `scalar` command' GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt,launchctl:true,schtasks:true" export GIT_TEST_MAINT_SCHEDULER +# Do not write any files outside the trash directory +Scalar_UNATTENDED=1 +export Scalar_UNATTENDED + +GIT_ASKPASS=true +export GIT_ASKPASS + test_expect_success 'scalar shows a usage' ' test_expect_code 129 scalar -h ' @@ -231,4 +238,165 @@ test_expect_success UNZIP 'scalar diagnose' ' grep "^Total: [1-9]" out ' +GIT_TEST_ALLOW_GVFS_VIA_HTTP=1 +export GIT_TEST_ALLOW_GVFS_VIA_HTTP + +test_set_port GIT_TEST_GVFS_PROTOCOL_PORT +HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT +PID_FILE="$(pwd)"/pid-file.pid +SERVER_LOG="$(pwd)"/OUT.server.log + +test_atexit ' + test -f "$PID_FILE" || return 0 + + # The server will shutdown automatically when we delete the pid-file. + rm -f "$PID_FILE" + + test -z "$verbose$verbose_log" || { + echo "server log:" + cat "$SERVER_LOG" + } + + # Give it a few seconds to shutdown (mainly to completely release the + # port before the next test start another instance and it attempts to + # bind to it). + for k in $(test_seq 5) + do + grep -q "Starting graceful shutdown" "$SERVER_LOG" && + return 0 || + sleep 1 + done + + echo "stop_gvfs_protocol_server: timeout waiting for server shutdown" + return 1 +' + +start_gvfs_enabled_http_server () { + GIT_HTTP_EXPORT_ALL=1 \ + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --reuseaddr \ + --pid-file="$PID_FILE" \ + 2>"$SERVER_LOG" & + + for k in 0 1 2 3 4 + do + if test -f "$PID_FILE" + then + return 0 + fi + sleep 1 + done + return 1 +} + +test_expect_success 'start GVFS-enabled server' ' + git config uploadPack.allowFilter false && + git config uploadPack.allowAnySHA1InWant false && + start_gvfs_enabled_http_server +' + +test_expect_success '`scalar clone` with GVFS-enabled server' ' + : the fake cache server requires fake authentication && + git config --global core.askPass true && + scalar clone --single-branch -- http://$HOST_PORT/ using-gvfs && + + : verify that the shared cache has been configured && + cache_key="url_$(printf "%s" http://$HOST_PORT/ | + tr A-Z a-z | + test-tool sha1)" && + echo "$(pwd)/.scalarCache/$cache_key" >expect && + git -C using-gvfs/src config gvfs.sharedCache >actual && + test_cmp expect actual && + + second=$(git rev-parse --verify second:second.t) && + ( + cd using-gvfs/src && + test_path_is_missing 1/2 && + GIT_TRACE=$PWD/trace.txt git cat-file blob $second >actual && + : verify that the gvfs-helper was invoked to fetch it && + test_i18ngrep gvfs-helper trace.txt && + echo "second" >expect && + test_cmp expect actual + ) +' + +test_expect_success '`scalar register` parallel to worktree is unsupported' ' + git init test-repo/src && + mkdir -p test-repo/out && + + : parallel to worktree is unsupported && + test_must_fail env GIT_CEILING_DIRECTORIES="$(pwd)" \ + scalar register test-repo/out && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/test-repo/src" scalar.repos && + + : at enlistment root, i.e. parent of repository, is supported && + GIT_CEILING_DIRECTORIES="$(pwd)" scalar register test-repo && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + grep -F "$(pwd)/test-repo/src" scalar.repos && + + : scalar delete properly unregisters enlistment && + scalar delete test-repo && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/test-repo/src" scalar.repos +' + +test_expect_success '`scalar register` & `unregister` with existing repo' ' + git init existing && + scalar register existing && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar unregister` with existing repo, deleted .git' ' + scalar register existing && + rm -rf existing/.git && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar register` existing repo with `src` folder' ' + git init existing && + mkdir -p existing/src && + scalar register existing/src && + scalar list >scalar.repos && + grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar delete` with existing repo' ' + git init existing && + scalar register existing && + scalar delete existing && + test_path_is_missing existing +' + +test_expect_success '`scalar clone --no-src`' ' + scalar clone --src "file://$(pwd)" with-src && + scalar clone --no-src "file://$(pwd)" without-src && + + test_path_is_dir with-src/src && + test_path_is_missing without-src/src +' + test_done