From ba808251aa9463ac192000316304a348e5eabfbe Mon Sep 17 00:00:00 2001 From: Shaoxuan Yuan Date: Sun, 7 Aug 2022 12:13:32 +0800 Subject: [PATCH 1/4] t1092: add tests for `git-rm` Add tests for `git-rm`, make sure it behaves as expected when is both inside or outside of sparse-checkout definition. Helped-by: Victoria Dye Helped-by: Derrick Stolee Signed-off-by: Shaoxuan Yuan Signed-off-by: Junio C Hamano --- t/t1092-sparse-checkout-compatibility.sh | 57 ++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 3588dd7b10..3ee2c9d792 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -1878,4 +1878,61 @@ test_expect_success 'mv directory from out-of-cone to in-cone' ' grep -e "H deep/0/1" actual ' +test_expect_success 'rm pathspec inside sparse definition' ' + init_repos && + + test_all_match git rm deep/a && + test_all_match git status --porcelain=v2 && + + # test wildcard + run_on_all git reset --hard && + test_all_match git rm deep/* && + test_all_match git status --porcelain=v2 && + + # test recursive rm + run_on_all git reset --hard && + test_all_match git rm -r deep && + test_all_match git status --porcelain=v2 +' + +test_expect_failure 'rm pathspec outside sparse definition' ' + init_repos && + + for file in folder1/a folder1/0/1 + do + test_sparse_match test_must_fail git rm $file && + test_sparse_match test_must_fail git rm --cached $file && + test_sparse_match git rm --sparse $file && + test_sparse_match git status --porcelain=v2 + done && + + cat >folder1-full <<-EOF && + rm ${SQ}folder1/0/0/0${SQ} + rm ${SQ}folder1/0/1${SQ} + rm ${SQ}folder1/a${SQ} + EOF + + cat >folder1-sparse <<-EOF && + rm ${SQ}folder1/${SQ} + EOF + + # test wildcard + run_on_sparse git reset --hard && + run_on_sparse git sparse-checkout reapply && + test_sparse_match test_must_fail git rm folder1/* && + run_on_sparse git rm --sparse folder1/* && + test_cmp folder1-full sparse-checkout-out && + test_cmp folder1-sparse sparse-index-out && + test_sparse_match git status --porcelain=v2 && + + # test recursive rm + run_on_sparse git reset --hard && + run_on_sparse git sparse-checkout reapply && + test_sparse_match test_must_fail git rm --sparse folder1 && + run_on_sparse git rm --sparse -r folder1 && + test_cmp folder1-full sparse-checkout-out && + test_cmp folder1-sparse sparse-index-out && + test_sparse_match git status --porcelain=v2 +' + test_done From b29ad38322d37925d217314383352c5341422a1c Mon Sep 17 00:00:00 2001 From: Shaoxuan Yuan Date: Sun, 7 Aug 2022 12:13:33 +0800 Subject: [PATCH 2/4] pathspec.h: move pathspec_needs_expanded_index() from reset.c to here Method pathspec_needs_expanded_index() in reset.c from 4d1cfc1351 (reset: make --mixed sparse-aware, 2021-11-29) is reusable when we need to verify if the index needs to be expanded when the command is utilizing a pathspec rather than a literal path. Move it to pathspec.h for reusability. Add a few items to the function so it can better serve its purpose as a standalone public function: * Add a check in front so if the index is not sparse, return early since no expansion is needed. * It now takes an arbitrary 'struct index_state' pointer instead of using `the_index` and `active_cache`. * Add documentation to the function. Helped-by: Victoria Dye Helped-by: Derrick Stolee Signed-off-by: Shaoxuan Yuan Signed-off-by: Junio C Hamano --- builtin/reset.c | 84 +--------------------------------------------- pathspec.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ pathspec.h | 12 +++++++ 3 files changed, 102 insertions(+), 83 deletions(-) diff --git a/builtin/reset.c b/builtin/reset.c index 344fff8f3a..fdce6f8c85 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -174,88 +174,6 @@ static void update_index_from_diff(struct diff_queue_struct *q, } } -static int pathspec_needs_expanded_index(const struct pathspec *pathspec) -{ - unsigned int i, pos; - int res = 0; - char *skip_worktree_seen = NULL; - - /* - * When using a magic pathspec, assume for the sake of simplicity that - * the index needs to be expanded to match all matchable files. - */ - if (pathspec->magic) - return 1; - - for (i = 0; i < pathspec->nr; i++) { - struct pathspec_item item = pathspec->items[i]; - - /* - * If the pathspec item has a wildcard, the index should be expanded - * if the pathspec has the possibility of matching a subset of entries inside - * of a sparse directory (but not the entire directory). - * - * If the pathspec item is a literal path, the index only needs to be expanded - * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't - * expand for in-cone files) and b) it doesn't match any sparse directories - * (since we can reset whole sparse directories without expanding them). - */ - if (item.nowildcard_len < item.len) { - /* - * Special case: if the pattern is a path inside the cone - * followed by only wildcards, the pattern cannot match - * partial sparse directories, so we know we don't need to - * expand the index. - * - * Examples: - * - in-cone/foo***: doesn't need expanded index - * - not-in-cone/bar*: may need expanded index - * - **.c: may need expanded index - */ - if (strspn(item.original + item.nowildcard_len, "*") == item.len - item.nowildcard_len && - path_in_cone_mode_sparse_checkout(item.original, &the_index)) - continue; - - for (pos = 0; pos < active_nr; pos++) { - struct cache_entry *ce = active_cache[pos]; - - if (!S_ISSPARSEDIR(ce->ce_mode)) - continue; - - /* - * If the pre-wildcard length is longer than the sparse - * directory name and the sparse directory is the first - * component of the pathspec, need to expand the index. - */ - if (item.nowildcard_len > ce_namelen(ce) && - !strncmp(item.original, ce->name, ce_namelen(ce))) { - res = 1; - break; - } - - /* - * If the pre-wildcard length is shorter than the sparse - * directory and the pathspec does not match the whole - * directory, need to expand the index. - */ - if (!strncmp(item.original, ce->name, item.nowildcard_len) && - wildmatch(item.original, ce->name, 0)) { - res = 1; - break; - } - } - } else if (!path_in_cone_mode_sparse_checkout(item.original, &the_index) && - !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) - res = 1; - - if (res > 0) - break; - } - - free(skip_worktree_seen); - return res; -} - static int read_from_tree(const struct pathspec *pathspec, struct object_id *tree_oid, int intent_to_add) @@ -273,7 +191,7 @@ static int read_from_tree(const struct pathspec *pathspec, opt.change = diff_change; opt.add_remove = diff_addremove; - if (pathspec->nr && the_index.sparse_index && pathspec_needs_expanded_index(pathspec)) + if (pathspec->nr && pathspec_needs_expanded_index(&the_index, pathspec)) ensure_full_index(&the_index); if (do_diff_cache(tree_oid, &opt)) diff --git a/pathspec.c b/pathspec.c index 84ad9c73cf..46e77a85fe 100644 --- a/pathspec.c +++ b/pathspec.c @@ -759,3 +759,92 @@ int match_pathspec_attrs(struct index_state *istate, return 1; } + +int pathspec_needs_expanded_index(struct index_state *istate, + const struct pathspec *pathspec) +{ + unsigned int i, pos; + int res = 0; + char *skip_worktree_seen = NULL; + + /* + * If index is not sparse, no index expansion is needed. + */ + if (!istate->sparse_index) + return 0; + + /* + * When using a magic pathspec, assume for the sake of simplicity that + * the index needs to be expanded to match all matchable files. + */ + if (pathspec->magic) + return 1; + + for (i = 0; i < pathspec->nr; i++) { + struct pathspec_item item = pathspec->items[i]; + + /* + * If the pathspec item has a wildcard, the index should be expanded + * if the pathspec has the possibility of matching a subset of entries inside + * of a sparse directory (but not the entire directory). + * + * If the pathspec item is a literal path, the index only needs to be expanded + * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't + * expand for in-cone files) and b) it doesn't match any sparse directories + * (since we can reset whole sparse directories without expanding them). + */ + if (item.nowildcard_len < item.len) { + /* + * Special case: if the pattern is a path inside the cone + * followed by only wildcards, the pattern cannot match + * partial sparse directories, so we know we don't need to + * expand the index. + * + * Examples: + * - in-cone/foo***: doesn't need expanded index + * - not-in-cone/bar*: may need expanded index + * - **.c: may need expanded index + */ + if (strspn(item.original + item.nowildcard_len, "*") == item.len - item.nowildcard_len && + path_in_cone_mode_sparse_checkout(item.original, istate)) + continue; + + for (pos = 0; pos < istate->cache_nr; pos++) { + struct cache_entry *ce = istate->cache[pos]; + + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + + /* + * If the pre-wildcard length is longer than the sparse + * directory name and the sparse directory is the first + * component of the pathspec, need to expand the index. + */ + if (item.nowildcard_len > ce_namelen(ce) && + !strncmp(item.original, ce->name, ce_namelen(ce))) { + res = 1; + break; + } + + /* + * If the pre-wildcard length is shorter than the sparse + * directory and the pathspec does not match the whole + * directory, need to expand the index. + */ + if (!strncmp(item.original, ce->name, item.nowildcard_len) && + wildmatch(item.original, ce->name, 0)) { + res = 1; + break; + } + } + } else if (!path_in_cone_mode_sparse_checkout(item.original, istate) && + !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) + res = 1; + + if (res > 0) + break; + } + + free(skip_worktree_seen); + return res; +} diff --git a/pathspec.h b/pathspec.h index 402ebb8080..41f6adfbb4 100644 --- a/pathspec.h +++ b/pathspec.h @@ -171,4 +171,16 @@ int match_pathspec_attrs(struct index_state *istate, const char *name, int namelen, const struct pathspec_item *item); +/* + * Determine whether a pathspec will match only entire index entries (non-sparse + * files and/or entire sparse directories). If the pathspec has the potential to + * match partial contents of a sparse directory, return 1 to indicate the index + * should be expanded to match the appropriate index entries. + * + * For the sake of simplicity, always return 1 if using a more complex "magic" + * pathspec. + */ +int pathspec_needs_expanded_index(struct index_state *istate, + const struct pathspec *pathspec); + #endif /* PATHSPEC_H */ From bcf96cfca6a9cfc62cea3998d9bb23d1c728a463 Mon Sep 17 00:00:00 2001 From: Shaoxuan Yuan Date: Sun, 7 Aug 2022 12:13:34 +0800 Subject: [PATCH 3/4] rm: expand the index only when necessary Remove the `ensure_full_index()` method so `git-rm` does not always expand the index when the expansion is unnecessary, i.e. when does not have any possibilities to match anything outside of sparse-checkout definition. Expand the index when the needs an expanded index, i.e. the contains wildcard that may need a full-index or the is simply outside of sparse-checkout definition. Notice that the test 'rm pathspec expands index when necessary' in t1092 *is* testing this code change behavior, though it will be marked as 'test_expect_success' only in the next patch, where we officially mark `command_requires_full_index = 0`, so the index does not expand unless we tell it to do so. Notice that because we also want `ensure_full_index` to record the stdout and stderr from Git command, a corresponding modification is also included in this patch. The reason we want the "sparse-index-out" and "sparse-index-err", is that we need to make sure there is no error from Git command itself, so we can rely on the `test_region` result and determine if the index is expanded or not. Helped-by: Victoria Dye Helped-by: Derrick Stolee Signed-off-by: Shaoxuan Yuan Signed-off-by: Junio C Hamano --- builtin/rm.c | 5 +++-- t/t1092-sparse-checkout-compatibility.sh | 27 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/builtin/rm.c b/builtin/rm.c index 84a935a16e..58ed924f0d 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -296,8 +296,9 @@ int cmd_rm(int argc, const char **argv, const char *prefix) seen = xcalloc(pathspec.nr, 1); - /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(&the_index); + if (pathspec_needs_expanded_index(&the_index, &pathspec)) + ensure_full_index(&the_index); + for (i = 0; i < active_nr; i++) { const struct cache_entry *ce = active_cache[i]; diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 3ee2c9d792..85247bb15f 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -1365,10 +1365,14 @@ ensure_not_expanded () { shift && test_must_fail env \ GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ - git -C sparse-index "$@" || return 1 + git -C sparse-index "$@" \ + >sparse-index-out \ + 2>sparse-index-error || return 1 else GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ - git -C sparse-index "$@" || return 1 + git -C sparse-index "$@" \ + >sparse-index-out \ + 2>sparse-index-error || return 1 fi && test_region ! index ensure_full_index trace2.txt } @@ -1935,4 +1939,23 @@ test_expect_failure 'rm pathspec outside sparse definition' ' test_sparse_match git status --porcelain=v2 ' +test_expect_failure 'rm pathspec expands index when necessary' ' + init_repos && + + # in-cone pathspec (do not expand) + ensure_not_expanded rm "deep/deep*" && + test_must_be_empty sparse-index-err && + + # out-of-cone pathspec (expand) + ! ensure_not_expanded rm --sparse "folder1/a*" && + test_must_be_empty sparse-index-err && + + # pathspec that should expand index + ! ensure_not_expanded rm "*/a" && + test_must_be_empty sparse-index-err && + + ! ensure_not_expanded rm "**a" && + test_must_be_empty sparse-index-err +' + test_done From ede241c7154929c5c80fb784daa79de6103bf048 Mon Sep 17 00:00:00 2001 From: Shaoxuan Yuan Date: Sun, 7 Aug 2022 12:13:35 +0800 Subject: [PATCH 4/4] rm: integrate with sparse-index Enable the sparse index within the `git-rm` command. The `p2000` tests demonstrate a ~92% execution time reduction for 'git rm' using a sparse index. Test HEAD~1 HEAD -------------------------------------------------------------------------- 2000.74: git rm ... (full-v3) 0.41(0.37+0.05) 0.43(0.36+0.07) +4.9% 2000.75: git rm ... (full-v4) 0.38(0.34+0.05) 0.39(0.35+0.05) +2.6% 2000.76: git rm ... (sparse-v3) 0.57(0.56+0.01) 0.05(0.05+0.00) -91.2% 2000.77: git rm ... (sparse-v4) 0.57(0.55+0.02) 0.03(0.03+0.00) -94.7% ---- Also, normalize a behavioral difference of `git-rm` under sparse-index. See related discussion [1]. `git-rm` a sparse-directory entry within a sparse-index enabled repo behaves differently from a sparse directory within a sparse-checkout enabled repo. For example, in a sparse-index repo, where 'folder1' is a sparse-directory entry, `git rm -r --sparse folder1` provides this: rm 'folder1/' Whereas in a sparse-checkout repo *without* sparse-index, doing so provides this: rm 'folder1/0/0/0' rm 'folder1/0/1' rm 'folder1/a' Because `git rm` a sparse-directory entry does not need to expand the index, therefore we should accept the current behavior, which is faster than "expand the sparse-directory entry to match the sparse-checkout situation". Modify a previous test so such difference is not considered as an error. [1] https://github.com/ffyuanda/git/pull/6#discussion_r934861398 Helped-by: Victoria Dye Helped-by: Derrick Stolee Signed-off-by: Shaoxuan Yuan Signed-off-by: Junio C Hamano --- builtin/rm.c | 2 ++ t/perf/p2000-sparse-operations.sh | 1 + t/t1092-sparse-checkout-compatibility.sh | 20 +++++++++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/builtin/rm.c b/builtin/rm.c index 58ed924f0d..b6ba859fe4 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -287,6 +287,8 @@ int cmd_rm(int argc, const char **argv, const char *prefix) if (!index_only) setup_work_tree(); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; hold_locked_index(&lock_file, LOCK_DIE_ON_ERROR); if (read_cache() < 0) diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index c181110a43..fce8151d41 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -123,5 +123,6 @@ test_perf_on_all git blame $SPARSE_CONE/f3/a test_perf_on_all git read-tree -mu HEAD test_perf_on_all git checkout-index -f --all test_perf_on_all git update-index --add --remove $SPARSE_CONE/a +test_perf_on_all "git rm -f $SPARSE_CONE/a && git checkout HEAD -- $SPARSE_CONE/a" test_done diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 85247bb15f..a6a14c8a21 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -937,7 +937,7 @@ test_expect_success 'read-tree --prefix' ' test_all_match git read-tree --prefix=deep/deeper1/deepest -u deepest && test_all_match git status --porcelain=v2 && - test_all_match git rm -rf --sparse folder1/ && + run_on_all git rm -rf --sparse folder1/ && test_all_match git read-tree --prefix=folder1/ -u update-folder1 && test_all_match git status --porcelain=v2 && @@ -1899,7 +1899,7 @@ test_expect_success 'rm pathspec inside sparse definition' ' test_all_match git status --porcelain=v2 ' -test_expect_failure 'rm pathspec outside sparse definition' ' +test_expect_success 'rm pathspec outside sparse definition' ' init_repos && for file in folder1/a folder1/0/1 @@ -1939,7 +1939,7 @@ test_expect_failure 'rm pathspec outside sparse definition' ' test_sparse_match git status --porcelain=v2 ' -test_expect_failure 'rm pathspec expands index when necessary' ' +test_expect_success 'rm pathspec expands index when necessary' ' init_repos && # in-cone pathspec (do not expand) @@ -1958,4 +1958,18 @@ test_expect_failure 'rm pathspec expands index when necessary' ' test_must_be_empty sparse-index-err ' +test_expect_success 'sparse index is not expanded: rm' ' + init_repos && + + ensure_not_expanded rm deep/a && + + # test in-cone wildcard + git -C sparse-index reset --hard && + ensure_not_expanded rm deep/* && + + # test recursive rm + git -C sparse-index reset --hard && + ensure_not_expanded rm -r deep +' + test_done