Merge branch 'sparse-index-stuff'

This is random stuff that probably all got upstream in the meantime.

Documentation/config/index.txt

@@ -1,3 +1,9 @@
+index.deleteSparseDirectories::
+	When enabled, the cone mode sparse-checkout feature will delete
+	directories that are outside of the sparse-checkout cone, unless
+	such a directory contains an untracked, non-ignored file. Defaults
+	to true.
+
 index.recordEndOfIndexEntries::
 	Specifies whether the index file should include an "End Of Index
 	Entry" section. This reduces index load time on multiprocessor

builtin/add.c

@@ -5,6 +5,7 @@
  */
 #define USE_THE_INDEX_VARIABLE
 #include "cache.h"
+#include "environment.h"
 #include "advice.h"
 #include "config.h"
 #include "builtin.h"
@@ -50,6 +51,7 @@ static int chmod_pathspec(struct pathspec *pathspec, char flip, int show_only)
 		int err;
 
 		if (!include_sparse &&
+		    !core_virtualfilesystem &&
 		    (ce_skip_worktree(ce) ||
 		     !path_in_sparse_checkout(ce->name, &the_index)))
 			continue;
@@ -100,7 +102,8 @@ static void update_callback(struct diff_queue_struct *q,
 		struct diff_filepair *p = q->queue[i];
 		const char *path = p->one->path;
 
-		if (!include_sparse && !path_in_sparse_checkout(path, &the_index))
+		if (!include_sparse && !core_virtualfilesystem &&
+		    !path_in_sparse_checkout(path, &the_index))
 			continue;
 
 		switch (fix_unmerged_status(p, data)) {
@@ -219,8 +222,9 @@ static int refresh(int verbose, const struct pathspec *pathspec)
 		if (!seen[i]) {
 			const char *path = pathspec->items[i].original;
 
-			if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
+			if (!core_virtualfilesystem &&
-			    !path_in_sparse_checkout(path, &the_index)) {
+			    (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
+			     !path_in_sparse_checkout(path, &the_index))) {
 				string_list_append(&only_match_skip_worktree,
 						   pathspec->items[i].original);
 			} else {
@@ -230,7 +234,11 @@ static int refresh(int verbose, const struct pathspec *pathspec)
 		}
 	}
 
-	if (only_match_skip_worktree.nr) {
+	/*
+	 * When using a virtual filesystem, we might re-add a path
+	 * that is currently virtual and we want that to succeed.
+	 */
+	if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
 		advise_on_updating_sparse_paths(&only_match_skip_worktree);
 		ret = 1;
 	}
@@ -604,7 +612,11 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 			if (seen[i])
 				continue;
 
-			if (!include_sparse &&
+			/*
+			 * When using a virtual filesystem, we might re-add a path
+			 * that is currently virtual and we want that to succeed.
+			 */
+			if (!include_sparse && !core_virtualfilesystem &&
 			    matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
 				string_list_append(&only_match_skip_worktree,
 						   pathspec.items[i].original);
@@ -628,7 +640,6 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 			}
 		}
 
-
 		if (only_match_skip_worktree.nr) {
 			advise_on_updating_sparse_paths(&only_match_skip_worktree);
 			exit_status = 1;

builtin/gc.c

@@ -1309,6 +1309,8 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts)
 	char *lock_path = xstrfmt("%s/maintenance", r->objects->odb->path);
 
 	if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) {
+		struct stat st;
+		struct strbuf lock_dot_lock = STRBUF_INIT;
 		/*
 		 * Another maintenance command is running.
 		 *
@@ -1319,6 +1321,25 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts)
 		if (!opts->auto_flag && !opts->quiet)
 			warning(_("lock file '%s' exists, skipping maintenance"),
 				lock_path);
+
+		/*
+		 * Check timestamp on .lock file to see if we should
+		 * delete it to recover from a fail state.
+		 */
+		strbuf_addstr(&lock_dot_lock, lock_path);
+		strbuf_addstr(&lock_dot_lock, ".lock");
+		if (lstat(lock_dot_lock.buf, &st))
+			warning_errno(_("unable to stat '%s'"), lock_dot_lock.buf);
+		else {
+			if (st.st_mtime < time(NULL) - (6 * 60 * 60)) {
+				if (unlink(lock_dot_lock.buf))
+					warning_errno(_("unable to delete stale lock file"));
+				else
+					warning(_("deleted stale lock file"));
+			}
+		}
+
+		strbuf_release(&lock_dot_lock);
 		free(lock_path);
 		return 0;
 	}

builtin/reset.c

@@ -37,6 +37,8 @@
 #include "add-interactive.h"
 #include "strbuf.h"
 #include "quote.h"
+#include "dir.h"
+#include "entry.h"
 
 #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000)
 
@@ -153,9 +155,47 @@ static void update_index_from_diff(struct diff_queue_struct *q,
 
 	for (i = 0; i < q->nr; i++) {
 		int pos;
+		int respect_skip_worktree = 1;
 		struct diff_filespec *one = q->queue[i]->one;
+		struct diff_filespec *two = q->queue[i]->two;
 		int is_in_reset_tree = one->mode && !is_null_oid(&one->oid);
+		int is_missing = !(one->mode && !is_null_oid(&one->oid));
+		int was_missing = !two->mode && is_null_oid(&two->oid);
 		struct cache_entry *ce;
+		struct cache_entry *ceBefore;
+		struct checkout state = CHECKOUT_INIT;
+
+		/*
+		 * When using the virtual filesystem feature, the cache entries that are
+		 * added here will not have the skip-worktree bit set.
+		 *
+		 * Without this code there is data that is lost because the files that
+		 * would normally be in the working directory are not there and show as
+		 * deleted for the next status or in the case of added files just disappear.
+		 * We need to create the previous version of the files in the working
+		 * directory so that they will have the right content and the next
+		 * status call will show modified or untracked files correctly.
+		 */
+		if (core_virtualfilesystem && !file_exists(two->path))
+		{
+			pos = index_name_pos(&the_index, two->path, strlen(two->path));
+			if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) &&
+			    (is_missing || !was_missing))
+			{
+				state.force = 1;
+				state.refresh_cache = 1;
+				state.istate = &the_index;
+				ceBefore = make_cache_entry(&the_index, two->mode,
+							    &two->oid, two->path,
+							    0, 0);
+				if (!ceBefore)
+					die(_("make_cache_entry failed for path '%s'"),
+						two->path);
+
+				checkout_entry(ceBefore, &state, NULL, NULL);
+				respect_skip_worktree = 0;
+			}
+		}
 
 		if (!is_in_reset_tree && !intent_to_add) {
 			remove_file_from_index(&the_index, one->path);
@@ -174,8 +214,14 @@ static void update_index_from_diff(struct diff_queue_struct *q,
 		 * to properly construct the reset sparse directory.
 		 */
 		pos = index_name_pos(&the_index, one->path, strlen(one->path));
-		if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) ||
+
-		    (pos < 0 && !path_in_sparse_checkout(one->path, &the_index)))
+		/*
+		 * Do not add the SKIP_WORKTREE bit back if we populated the
+		 * file on purpose in a virtual filesystem scenario.
+		 */
+		if (respect_skip_worktree &&
+		    ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) ||
+		     (pos < 0 && !path_in_sparse_checkout(one->path, &the_index))))
 			ce->ce_flags |= CE_SKIP_WORKTREE;
 
 		if (!ce)

builtin/rm.c

@@ -5,6 +5,7 @@
  */
 #define USE_THE_INDEX_VARIABLE
 #include "builtin.h"
+#include "environment.h"
 #include "alloc.h"
 #include "advice.h"
 #include "config.h"
@@ -311,7 +312,7 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
 	for (i = 0; i < the_index.cache_nr; i++) {
 		const struct cache_entry *ce = the_index.cache[i];
 
-		if (!include_sparse &&
+		if (!include_sparse && !core_virtualfilesystem &&
 		    (ce_skip_worktree(ce) ||
 		     !path_in_sparse_checkout(ce->name, &the_index)))
 			continue;
@@ -348,7 +349,11 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
 				    *original ? original : ".");
 		}
 
-		if (only_match_skip_worktree.nr) {
+		/*
+		 * When using a virtual filesystem, we might re-add a path
+		 * that is currently virtual and we want that to succeed.
+		 */
+		if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
 			advise_on_updating_sparse_paths(&only_match_skip_worktree);
 			ret = 1;
 		}

builtin/sparse-checkout.c

@@ -112,7 +112,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix)
 
 static void clean_tracked_sparse_directories(struct repository *r)
 {
-	int i, was_full = 0;
+	int i, value, was_full = 0;
 	struct strbuf path = STRBUF_INIT;
 	size_t pathlen;
 	struct string_list_item *item;
@@ -128,6 +128,13 @@ static void clean_tracked_sparse_directories(struct repository *r)
 	    !r->index->sparse_checkout_patterns->use_cone_patterns)
 		return;
 
+	/*
+	 * Users can disable this behavior.
+	 */
+	if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) &&
+	    !value)
+		return;
+
 	/*
 	 * Use the sparse index as a data structure to assist finding
 	 * directories that are safe to delete. This conversion to a

diff.c

@@ -3992,6 +3992,13 @@ static int reuse_worktree_file(struct index_state *istate,
 	if (!FAST_WORKING_DIRECTORY && !want_file && has_object_pack(oid))
 		return 0;
 
+	/*
+	 * If this path does not match our sparse-checkout definition,
+	 * then the file will not be in the working directory.
+	 */
+	if (!path_in_sparse_checkout(name, istate))
+		return 0;
+
 	/*
 	 * Similarly, if we'd have to convert the file contents anyway, that
 	 * makes the optimization not worthwhile.

dir.c

@@ -1531,6 +1531,13 @@ static int path_in_sparse_checkout_1(const char *path,
 	enum pattern_match_result match = UNDECIDED;
 	const char *end, *slash;
 
+	/*
+	 * When using a virtual filesystem, there aren't really patterns
+	 * to follow, but be extra careful to skip this check.
+	 */
+	if (core_virtualfilesystem)
+		return 1;
+
 	/*
 	 * We default to accepting a path if the path is empty, there are no
 	 * patterns, or the patterns are of the wrong type.

repo-settings.c

@@ -86,7 +86,7 @@ void prepare_repo_settings(struct repository *r)
 	/* Boolean config or default, does not cascade (simple)  */
 	repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1);
 	repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1);
-	repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0);
+	repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 1);
 	repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash);
 	repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1);
 

sequencer.c

@@ -696,7 +696,7 @@ static int do_recursive_merge(struct repository *r,
 	o.branch2 = next ? next_label : "(empty tree)";
 	if (is_rebase_i(opts))
 		o.buffer_output = 2;
-	o.show_rename_progress = 1;
+	o.show_rename_progress = isatty(2);
 
 	head_tree = parse_tree_indirect(head);
 	next_tree = next ? repo_get_commit_tree(r, next) : empty_tree(r);

t/perf/p2000-sparse-operations.sh

@@ -56,7 +56,7 @@ test_expect_success 'setup repo and indexes' '
 	git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v3 &&
 	(
 		cd full-v3 &&
-		git sparse-checkout init --cone &&
+		git sparse-checkout init --cone --no-sparse-index &&
 		git sparse-checkout set $SPARSE_CONE &&
 		git config index.version 3 &&
 		git update-index --index-version=3 &&
@@ -65,7 +65,7 @@ test_expect_success 'setup repo and indexes' '
 	git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v4 &&
 	(
 		cd full-v4 &&
-		git sparse-checkout init --cone &&
+		git sparse-checkout init --cone --no-sparse-index &&
 		git sparse-checkout set $SPARSE_CONE &&
 		git config index.version 4 &&
 		git update-index --index-version=4 &&

t/t1091-sparse-checkout-builtin.sh

@@ -782,6 +782,10 @@ test_expect_success 'cone mode clears ignored subdirectories' '
 	git -C repo status --porcelain=v2 >out &&
 	test_must_be_empty out &&
 
+	git -C repo -c index.deleteSparseDirectories=false sparse-checkout reapply &&
+	test_path_is_dir repo/folder1 &&
+	test_path_is_dir repo/deep/deeper2 &&
+
 	git -C repo sparse-checkout reapply &&
 	test_path_is_missing repo/folder1 &&
 	test_path_is_missing repo/deep/deeper2 &&

t/t1092-sparse-checkout-compatibility.sh

@@ -155,6 +155,7 @@ init_repos () {
 	git -C sparse-index reset --hard &&
 
 	# initialize sparse-checkout definitions
+	git -C sparse-checkout config index.sparse false &&
 	git -C sparse-checkout sparse-checkout init --cone &&
 	git -C sparse-checkout sparse-checkout set deep &&
 	git -C sparse-index sparse-checkout init --cone --sparse-index &&
@@ -517,6 +518,43 @@ test_expect_success 'diff --cached' '
 	test_all_match git diff --cached
 '
 
+test_expect_success 'diff partially-staged' '
+	init_repos &&
+
+	write_script edit-contents <<-\EOF &&
+	echo text >>$1
+	EOF
+
+	# Add file within cone
+	test_all_match git sparse-checkout set deep &&
+	run_on_all ../edit-contents deep/testfile &&
+	test_all_match git add deep/testfile &&
+	run_on_all ../edit-contents deep/testfile &&
+
+	test_all_match git diff &&
+	test_all_match git diff --staged &&
+
+	# Add file outside cone
+	test_all_match git reset --hard &&
+	run_on_all mkdir newdirectory &&
+	run_on_all ../edit-contents newdirectory/testfile &&
+	test_all_match git sparse-checkout set newdirectory &&
+	test_all_match git add newdirectory/testfile &&
+	run_on_all ../edit-contents newdirectory/testfile &&
+	test_all_match git sparse-checkout set &&
+
+	test_all_match git diff &&
+	test_all_match git diff --staged &&
+
+	# Merge conflict outside cone
+	test_all_match git reset --hard &&
+	test_all_match git checkout merge-left &&
+	test_all_match test_must_fail git merge merge-right &&
+
+	test_all_match git diff &&
+	test_all_match git diff --staged
+'
+
 # NEEDSWORK: sparse-checkout behaves differently from full-checkout when
 # running this test with 'df-conflict-2' after 'df-conflict-1'.
 test_expect_success 'diff with renames and conflicts' '
@@ -991,7 +1029,9 @@ test_expect_success 'read-tree --merge with directory-file conflicts' '
 test_expect_success 'merge, cherry-pick, and rebase' '
 	init_repos &&
 
-	for OPERATION in "merge -m merge" cherry-pick "rebase --apply" "rebase --merge"
+	# microsoft/git specific: we need to use "quiet" mode
+	# to avoid different stderr for some rebases.
+	for OPERATION in "merge -m merge" cherry-pick "rebase -q --apply" "rebase -q --merge"
 	do
 		test_all_match git checkout -B temp update-deep &&
 		test_all_match git $OPERATION update-folder1 &&
@@ -1452,6 +1492,11 @@ test_expect_success 'sparse-index is not expanded' '
 	ensure_not_expanded reset --merge update-deep &&
 	ensure_not_expanded reset --hard &&
 
+	echo a test change >>sparse-index/README.md &&
+	ensure_not_expanded diff &&
+	git -C sparse-index add README.md &&
+	ensure_not_expanded diff --staged &&
+
 	ensure_not_expanded reset base -- deep/a &&
 	ensure_not_expanded reset base -- nonexistent-file &&
 	ensure_not_expanded reset deepest -- deep &&
@@ -1772,6 +1817,46 @@ test_expect_success 'sparse index is not expanded: sparse-checkout' '
 	ensure_not_expanded sparse-checkout set
 '
 
+# NEEDSWORK: although the full repository's index is _not_ expanded as part of
+# stash, a temporary index, which is _not_ sparse, is created when stashing and
+# applying a stash of untracked files. As a result, the test reports that it
+# finds an instance of `ensure_full_index`, but it does not carry with it the
+# performance implications of expanding the full repository index.
+test_expect_success 'sparse index is not expanded: stash -u' '
+	init_repos &&
+
+	mkdir -p sparse-index/folder1 &&
+	echo >>sparse-index/README.md &&
+	echo >>sparse-index/a &&
+	echo >>sparse-index/folder1/new &&
+
+	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
+		git -C sparse-index stash -u &&
+	test_region index ensure_full_index trace2.txt &&
+
+	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
+		git -C sparse-index stash pop &&
+	test_region index ensure_full_index trace2.txt
+'
+
+# NEEDSWORK: similar to `git add`, untracked files outside of the sparse
+# checkout definition are successfully stashed and unstashed.
+test_expect_success 'stash -u outside sparse checkout definition' '
+	init_repos &&
+
+	write_script edit-contents <<-\EOF &&
+	echo text >>$1
+	EOF
+
+	run_on_sparse mkdir -p folder1 &&
+	run_on_all ../edit-contents folder1/new &&
+	test_all_match git stash -u &&
+	test_all_match git status --porcelain=v2 &&
+
+	test_all_match git stash pop -q &&
+	test_all_match git status --porcelain=v2
+'
+
 # NEEDSWORK: a sparse-checkout behaves differently from a full checkout
 # in this scenario, but it shouldn't.
 test_expect_success 'reset mixed and checkout orphan' '

t/t7817-grep-sparse-checkout.sh

@@ -49,7 +49,7 @@ test_expect_success 'setup' '
 		echo "text" >B/b &&
 		git add A B &&
 		git commit -m sub &&
-		git sparse-checkout init --cone &&
+		git sparse-checkout init --cone --no-sparse-index &&
 		git sparse-checkout set B
 	) &&
 

t/t7900-maintenance.sh

@@ -54,6 +54,23 @@ test_expect_success 'run [--auto|--quiet]' '
 	test_subcommand git gc --no-quiet <run-no-quiet.txt
 '
 
+test_expect_success 'lock file behavior' '
+	test_when_finished git config --unset maintenance.commit-graph.schedule &&
+	git config maintenance.commit-graph.schedule hourly &&
+
+	touch .git/objects/maintenance.lock &&
+	git maintenance run --schedule=hourly --no-quiet 2>err &&
+	grep "lock file .* exists, skipping maintenance" err &&
+
+	test-tool chmtime =-22000 .git/objects/maintenance.lock &&
+	git maintenance run --schedule=hourly --no-quiet 2>err &&
+	grep "deleted stale lock file" err &&
+	test_path_is_missing .git/objects/maintenance.lock &&
+
+	git maintenance run --schedule=hourly 2>err &&
+	test_must_be_empty err
+'
+
 test_expect_success 'maintenance.auto config option' '
 	GIT_TRACE2_EVENT="$(pwd)/default" git commit --quiet --allow-empty -m 1 &&
 	test_subcommand git maintenance run --auto --quiet <default &&