From 522e6417487cc5c3f2f6d49c8f63554af63d8eda Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:31 +0000 Subject: [PATCH 01/19] t1091: use check_files to reduce boilerplate When testing the sparse-checkout feature, we need to compare the contents of the working-directory against some expected output. Using here-docs was useful in the beginning, but became repetetive as the test script grew. Create a check_files helper to make the tests simpler and easier to extend. It also reduces instances of bad here-doc whitespace. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 117 ++++++----------------------- 1 file changed, 22 insertions(+), 95 deletions(-) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index ff7f8f7a1f..e058a20ad6 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -12,6 +12,13 @@ list_files() { (cd "$1" && printf '%s\n' *) } +check_files() { + list_files "$1" >actual && + shift && + printf "%s\n" $@ >expect && + test_cmp expect actual +} + test_expect_success 'setup' ' git init repo && ( @@ -58,9 +65,7 @@ test_expect_success 'git sparse-checkout init' ' EOF test_cmp expect repo/.git/info/sparse-checkout && test_cmp_config -C repo true core.sparsecheckout && - list_files repo >dir && - echo a >expect && - test_cmp expect dir + check_files repo a ' test_expect_success 'git sparse-checkout list after init' ' @@ -81,13 +86,7 @@ test_expect_success 'init with existing sparse-checkout' ' *folder* EOF test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'clone --sparse' ' @@ -98,9 +97,7 @@ test_expect_success 'clone --sparse' ' !/*/ EOF test_cmp expect actual && - list_files clone >dir && - echo a >expect && - test_cmp expect dir + check_files clone a ' test_expect_success 'set enables config' ' @@ -127,13 +124,7 @@ test_expect_success 'set sparse-checkout using builtin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'set sparse-checkout using --stdin' ' @@ -147,13 +138,7 @@ test_expect_success 'set sparse-checkout using --stdin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo "a folder1 folder2" ' test_expect_success 'cone mode: match patterns' ' @@ -162,13 +147,7 @@ test_expect_success 'cone mode: match patterns' ' git -C repo read-tree -mu HEAD 2>err && test_i18ngrep ! "disabling cone patterns" err && git -C repo reset --hard && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'cone mode: warn on bad pattern' ' @@ -185,14 +164,7 @@ test_expect_success 'sparse-checkout disable' ' test_path_is_file repo/.git/info/sparse-checkout && git -C repo config --list >config && test_must_fail git config core.sparseCheckout && - list_files repo >dir && - cat >expect <<-EOF && - a - deep - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a deep folder1 folder2 ' test_expect_success 'cone mode: init and set' ' @@ -204,24 +176,9 @@ test_expect_success 'cone mode: init and set' ' test_cmp expect dir && git -C repo sparse-checkout set deep/deeper1/deepest/ 2>err && test_must_be_empty err && - list_files repo >dir && - cat >expect <<-EOF && - a - deep - EOF - test_cmp expect dir && - list_files repo/deep >dir && - cat >expect <<-EOF && - a - deeper1 - EOF - test_cmp expect dir && - list_files repo/deep/deeper1 >dir && - cat >expect <<-EOF && - a - deepest - EOF - test_cmp expect dir && + check_files repo a deep && + check_files repo/deep a deeper1 && + check_files repo/deep/deeper1 a deepest && cat >expect <<-EOF && /* !/*/ @@ -237,13 +194,7 @@ test_expect_success 'cone mode: init and set' ' folder2 EOF test_must_be_empty err && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - list_files repo >dir && - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'cone mode: list' ' @@ -275,13 +226,7 @@ test_expect_success 'revert to old sparse-checkout on bad update' ' test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && test_i18ngrep "cannot set sparse-checkout patterns" err && test_cmp repo/.git/info/sparse-checkout expect && - list_files repo/deep >dir && - cat >expect <<-EOF && - a - deeper1 - deeper2 - EOF - test_cmp dir expect + check_files repo/deep a deeper1 deeper2 ' test_expect_success 'revert to old sparse-checkout on empty update' ' @@ -332,12 +277,7 @@ test_expect_success 'cone mode: set with core.ignoreCase=true' ' /folder1/ EOF test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - EOF - test_cmp expect dir + check_files repo a folder1 ' test_expect_success 'interaction with submodules' ' @@ -351,21 +291,8 @@ test_expect_success 'interaction with submodules' ' git sparse-checkout init --cone && git sparse-checkout set folder1 ) && - list_files super >dir && - cat >expect <<-\EOF && - a - folder1 - modules - EOF - test_cmp expect dir && - list_files super/modules/child >dir && - cat >expect <<-\EOF && - a - deep - folder1 - folder2 - EOF - test_cmp expect dir + check_files super a folder1 modules && + check_files super/modules/child a deep folder1 folder2 ' test_done From d622c34396b3ea1a81f07d951ee1112f83d9330c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:32 +0000 Subject: [PATCH 02/19] t1091: improve here-docs t1091-sparse-checkout-builtin.sh uses here-docs to populate the expected contents of the sparse-checkout file. These do not use shell interpolation, so use "-\EOF" instead of "-EOF". Also use proper tabbing. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 98 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e058a20ad6..e28e1c797f 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -46,11 +46,11 @@ test_expect_success 'git sparse-checkout list (empty)' ' test_expect_success 'git sparse-checkout list (populated)' ' test_when_finished rm -f repo/.git/info/sparse-checkout && - cat >repo/.git/info/sparse-checkout <<-EOF && - /folder1/* - /deep/ - **/a - !*bin* + cat >repo/.git/info/sparse-checkout <<-\EOF && + /folder1/* + /deep/ + **/a + !*bin* EOF cp repo/.git/info/sparse-checkout expect && git -C repo sparse-checkout list >list && @@ -59,9 +59,9 @@ test_expect_success 'git sparse-checkout list (populated)' ' test_expect_success 'git sparse-checkout init' ' git -C repo sparse-checkout init && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect repo/.git/info/sparse-checkout && test_cmp_config -C repo true core.sparsecheckout && @@ -70,9 +70,9 @@ test_expect_success 'git sparse-checkout init' ' test_expect_success 'git sparse-checkout list after init' ' git -C repo sparse-checkout list >actual && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect actual ' @@ -80,10 +80,10 @@ test_expect_success 'git sparse-checkout list after init' ' test_expect_success 'init with existing sparse-checkout' ' echo "*folder*" >> repo/.git/info/sparse-checkout && git -C repo sparse-checkout init && - cat >expect <<-EOF && - /* - !/*/ - *folder* + cat >expect <<-\EOF && + /* + !/*/ + *folder* EOF test_cmp expect repo/.git/info/sparse-checkout && check_files repo a folder1 folder2 @@ -92,9 +92,9 @@ test_expect_success 'init with existing sparse-checkout' ' test_expect_success 'clone --sparse' ' git clone --sparse repo clone && git -C clone sparse-checkout list >actual && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect actual && check_files clone a @@ -116,10 +116,10 @@ test_expect_success 'set enables config' ' test_expect_success 'set sparse-checkout using builtin' ' git -C repo sparse-checkout set "/*" "!/*/" "*folder*" && - cat >expect <<-EOF && - /* - !/*/ - *folder* + cat >expect <<-\EOF && + /* + !/*/ + *folder* EOF git -C repo sparse-checkout list >actual && test_cmp expect actual && @@ -128,11 +128,11 @@ test_expect_success 'set sparse-checkout using builtin' ' ' test_expect_success 'set sparse-checkout using --stdin' ' - cat >expect <<-EOF && - /* - !/*/ - /folder1/ - /folder2/ + cat >expect <<-\EOF && + /* + !/*/ + /folder1/ + /folder2/ EOF git -C repo sparse-checkout set --stdin actual && @@ -179,28 +179,28 @@ test_expect_success 'cone mode: init and set' ' check_files repo a deep && check_files repo/deep a deeper1 && check_files repo/deep/deeper1 a deepest && - cat >expect <<-EOF && - /* - !/*/ - /deep/ - !/deep/*/ - /deep/deeper1/ - !/deep/deeper1/*/ - /deep/deeper1/deepest/ + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + !/deep/deeper1/*/ + /deep/deeper1/deepest/ EOF test_cmp expect repo/.git/info/sparse-checkout && - git -C repo sparse-checkout set --stdin 2>err <<-EOF && - folder1 - folder2 + git -C repo sparse-checkout set --stdin 2>err <<-\EOF && + folder1 + folder2 EOF test_must_be_empty err && check_files repo a folder1 folder2 ' test_expect_success 'cone mode: list' ' - cat >expect <<-EOF && - folder1 - folder2 + cat >expect <<-\EOF && + folder1 + folder2 EOF git -C repo sparse-checkout set --stdin actual 2>err && @@ -211,10 +211,10 @@ test_expect_success 'cone mode: list' ' test_expect_success 'cone mode: set with nested folders' ' git -C repo sparse-checkout set deep deep/deeper1/deepest 2>err && test_line_count = 0 err && - cat >expect <<-EOF && - /* - !/*/ - /deep/ + cat >expect <<-\EOF && + /* + !/*/ + /deep/ EOF test_cmp repo/.git/info/sparse-checkout expect ' @@ -271,10 +271,10 @@ test_expect_success 'sparse-checkout (init|set|disable) fails with dirty status' test_expect_success 'cone mode: set with core.ignoreCase=true' ' git -C repo sparse-checkout init --cone && git -C repo -c core.ignoreCase=true sparse-checkout set folder1 && - cat >expect <<-EOF && - /* - !/*/ - /folder1/ + cat >expect <<-\EOF && + /* + !/*/ + /folder1/ EOF test_cmp expect repo/.git/info/sparse-checkout && check_files repo a folder1 From 3c754067a1164ffafd965dcd44a9f004e6100e42 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:33 +0000 Subject: [PATCH 03/19] sparse-checkout: create leading directories The 'git init' command creates the ".git/info" directory and fills it with some default files. However, 'git worktree add' does not create the info directory for that worktree. This causes a problem when running "git sparse-checkout init" inside a worktree. While care was taken to allow the sparse-checkout config to be specific to a worktree, this initialization was untested. Safely create the leading directories for the sparse-checkout file. This is the safest thing to do even without worktrees, as a user could delete their ".git/info" directory and expect Git to recover safely. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 4 ++++ t/t1091-sparse-checkout-builtin.sh | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index b3bed891cb..3cee8ab46e 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -199,6 +199,10 @@ static int write_patterns_and_update(struct pattern_list *pl) int result; sparse_filename = get_sparse_checkout_filename(); + + if (safe_create_leading_directories(sparse_filename)) + die(_("failed to create directory for sparse-checkout file")); + fd = hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e28e1c797f..43d1f7520c 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -295,4 +295,14 @@ test_expect_success 'interaction with submodules' ' check_files super/modules/child a deep folder1 folder2 ' +test_expect_success 'different sparse-checkouts with worktrees' ' + git -C repo worktree add --detach ../worktree && + check_files worktree "a deep folder1 folder2" && + git -C worktree sparse-checkout init --cone && + git -C repo sparse-checkout set folder1 && + git -C worktree sparse-checkout set deep/deeper1 && + check_files repo a folder1 && + check_files worktree a deep +' + test_done From 47dbf10d8a5ce8c9b441a16b7698c7d70585dff0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:34 +0000 Subject: [PATCH 04/19] clone: fix --sparse option with URLs The --sparse option was added to the clone builtin in d89f09c (clone: add --sparse mode, 2019-11-21) and was tested with a local path clone in t1091-sparse-checkout-builtin.sh. However, due to a difference in how local paths are handled versus URLs, this mechanism does not work with URLs. Modify the test to use a "file://" URL, which would output this error before the code change: Cloning into 'clone'... fatal: cannot change to 'file://.../repo': No such file or directory error: failed to initialize sparse-checkout These errors are due to using a "-C " option to call 'git -C sparse-checkout init' but the URL is being given instead of the target directory. Update that target directory to evaluate this correctly. I have also manually tested that https:// URLs are handled correctly as well. Acked-by: Taylor Blau Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/clone.c | 2 +- t/t1091-sparse-checkout-builtin.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 4348d962c9..2caefc44fb 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1130,7 +1130,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_required_reference.nr || option_optional_reference.nr) setup_reference(); - if (option_sparse_checkout && git_sparse_checkout_init(repo)) + if (option_sparse_checkout && git_sparse_checkout_init(dir)) return 1; remote = remote_get(option_origin); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 43d1f7520c..cf4a595c86 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -90,7 +90,7 @@ test_expect_success 'init with existing sparse-checkout' ' ' test_expect_success 'clone --sparse' ' - git clone --sparse repo clone && + git clone --sparse "file://$(pwd)/repo" clone && git -C clone sparse-checkout list >actual && cat >expect <<-\EOF && /* From 7aa9ef2fcaa986d7f11064adab6d1c010d4f2ead Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 24 Jan 2020 21:19:35 +0000 Subject: [PATCH 05/19] sparse-checkout: fix documentation typo for core.sparseCheckoutCone Signed-off-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-sparse-checkout.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 3b341cf0fc..4834fb434d 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -106,7 +106,7 @@ The full pattern set allows for arbitrary pattern matches and complicated inclusion/exclusion rules. These can result in O(N*M) pattern matches when updating the index, where N is the number of patterns and M is the number of paths in the index. To combat this performance issue, a more restricted -pattern set is allowed when `core.spareCheckoutCone` is enabled. +pattern set is allowed when `core.sparseCheckoutCone` is enabled. The accepted patterns in the cone pattern set are: From 41de0c6fbcc3d2544ebada3a9f26dec0f32f42de Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:36 +0000 Subject: [PATCH 06/19] sparse-checkout: cone mode does not recognize "**" When core.sparseCheckoutCone is enabled, the 'git sparse-checkout set' command creates a restricted set of possible patterns that are used by a custom algorithm to quickly match those patterns. If a user manually edits the sparse-checkout file, then they could create patterns that do not match these expectations. The cone-mode matching algorithm can return incorrect results. The solution is to detect these incorrect patterns, warn that we do not recognize them, and revert to the standard algorithm. Check each pattern for the "**" substring, and revert to the old logic if seen. While technically a "//**" pattern matches the meaning of "//", it is not one that would be written by the sparse-checkout builtin in cone mode. Attempting to accept that pattern change complicates the logic and instead we punt and do not accept any instance of "**". Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- dir.c | 7 +++++- t/t1091-sparse-checkout-builtin.sh | 34 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index 22d08e61c2..40fed73a94 100644 --- a/dir.c +++ b/dir.c @@ -651,11 +651,16 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern return; } + if (strstr(given->pattern, "**")) { + /* Not a cone pattern. */ + warning(_("unrecognized pattern: '%s'"), given->pattern); + goto clear_hashmaps; + } + if (given->patternlen > 2 && !strcmp(given->pattern + given->patternlen - 2, "/*")) { if (!(given->flags & PATTERN_FLAG_NEGATIVE)) { /* Not a cone pattern. */ - pl->use_cone_patterns = 0; warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index cf4a595c86..e2e45dc7fd 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -305,4 +305,38 @@ test_expect_success 'different sparse-checkouts with worktrees' ' check_files worktree a deep ' +check_read_tree_errors () { + REPO=$1 + FILES=$2 + ERRORS=$3 + git -C $REPO read-tree -mu HEAD 2>err && + if test -z "$ERRORS" + then + test_must_be_empty err + else + test_i18ngrep "$ERRORS" err + fi && + check_files $REPO $FILES +} + +test_expect_success 'pattern-checks: /A/**' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /folder1/** + EOF + check_read_tree_errors repo "a folder1" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: /A/**/B/' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /deep/**/deepest + EOF + check_read_tree_errors repo "a deep" "disabling cone pattern matching" && + check_files repo/deep "deeper1" && + check_files repo/deep/deeper1 "deepest" +' + test_done From 9e6d3e64175713bc0007f3012ea288f4dfc0a399 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:37 +0000 Subject: [PATCH 07/19] sparse-checkout: detect short patterns In cone mode, the shortest pattern the sparse-checkout command will write into the sparse-checkout file is "/*". This is handled carefully in add_pattern_to_hashsets(), so warn if any other pattern is this short. This will assist future pattern checks by allowing us to assume there are at least three characters in the pattern. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- dir.c | 3 ++- t/t1091-sparse-checkout-builtin.sh | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index 40fed73a94..c2e585607e 100644 --- a/dir.c +++ b/dir.c @@ -651,7 +651,8 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern return; } - if (strstr(given->pattern, "**")) { + if (given->patternlen <= 2 || + strstr(given->pattern, "**")) { /* Not a cone pattern. */ warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e2e45dc7fd..2e57534799 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -339,4 +339,13 @@ test_expect_success 'pattern-checks: /A/**/B/' ' check_files repo/deep/deeper1 "deepest" ' +test_expect_success 'pattern-checks: too short' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /a + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" +' + test_done From 9abc60f8015d060d3f3433b105648a4725c97bd1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:08 +0000 Subject: [PATCH 08/19] sparse-checkout: warn on globs in cone patterns In cone mode, the sparse-checkout commmand will write patterns that allow faster pattern matching. This matching only works if the patterns in the sparse-checkout file are those written by that command. Users can edit the sparse-checkout file and create patterns that cause the cone mode matching to fail. The cone mode patterns may end in "/*" but otherwise an un-escaped asterisk or other glob character is invalid. Add checks to disable cone mode when seeing these values. A later change will properly handle escaped globs. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- dir.c | 36 +++++++++++++++++++++++++++ t/t1091-sparse-checkout-builtin.sh | 39 ++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/dir.c b/dir.c index c2e585607e..71d28331f3 100644 --- a/dir.c +++ b/dir.c @@ -635,6 +635,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern struct pattern_entry *translated; char *truncated; char *data = NULL; + const char *prev, *cur, *next; if (!pl->use_cone_patterns) return; @@ -652,12 +653,47 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern } if (given->patternlen <= 2 || + *given->pattern == '*' || strstr(given->pattern, "**")) { /* Not a cone pattern. */ warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; } + prev = given->pattern; + cur = given->pattern + 1; + next = given->pattern + 2; + + while (*cur) { + /* Watch for glob characters '*', '\', '[', '?' */ + if (!is_glob_special(*cur)) + goto increment; + + /* But only if *prev != '\\' */ + if (*prev == '\\') + goto increment; + + /* But allow the initial '\' */ + if (*cur == '\\' && + is_glob_special(*next)) + goto increment; + + /* But a trailing '/' then '*' is fine */ + if (*prev == '/' && + *cur == '*' && + *next == 0) + goto increment; + + /* Not a cone pattern. */ + warning(_("unrecognized pattern: '%s'"), given->pattern); + goto clear_hashmaps; + + increment: + prev++; + cur++; + next++; + } + if (given->patternlen > 2 && !strcmp(given->pattern + given->patternlen - 2, "/*")) { if (!(given->flags & PATTERN_FLAG_NEGATIVE)) { diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 2e57534799..c732abeacd 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -348,4 +348,43 @@ test_expect_success 'pattern-checks: too short' ' check_read_tree_errors repo "a" "disabling cone pattern matching" ' +test_expect_success 'pattern-checks: trailing "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /a* + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: starting "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + *eep/ + EOF + check_read_tree_errors repo "a deep" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: contained glob characters' ' + for c in "[a]" "\\" "?" "*" + do + cat >repo/.git/info/sparse-checkout <<-EOF && + /* + !/*/ + something$c-else/ + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" + done +' + +test_expect_success 'pattern-checks: escaped "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /does\*not\*exist/ + EOF + check_read_tree_errors repo "a" "" +' + test_done From 4f52c2ce6c578896964e960f6017510f0efd3f46 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:09 +0000 Subject: [PATCH 09/19] sparse-checkout: properly match escaped characters In cone mode, the sparse-checkout feature uses hashset containment queries to match paths. Make this algorithm respect escaped asterisk (*) and backslash (\) characters. Create dup_and_filter_pattern() method to convert a pattern by removing escape characters and dropping an optional "/*" at the end. This method is available in dir.h as we will use it in builtin/sparse-checkout.c in a later change. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- dir.c | 35 +++++++++++++++++++++++++++--- t/t1091-sparse-checkout-builtin.sh | 23 ++++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/dir.c b/dir.c index 71d28331f3..7ac0920b71 100644 --- a/dir.c +++ b/dir.c @@ -630,6 +630,36 @@ int pl_hashmap_cmp(const void *unused_cmp_data, return strncmp(ee1->pattern, ee2->pattern, min_len); } +static char *dup_and_filter_pattern(const char *pattern) +{ + char *set, *read; + size_t count = 0; + char *result = xstrdup(pattern); + + set = result; + read = result; + + while (*read) { + /* skip escape characters (once) */ + if (*read == '\\') + read++; + + *set = *read; + + set++; + read++; + count++; + } + *set = 0; + + if (count > 2 && + *(set - 1) == '*' && + *(set - 2) == '/') + *(set - 2) = 0; + + return result; +} + static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern *given) { struct pattern_entry *translated; @@ -702,8 +732,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern goto clear_hashmaps; } - truncated = xstrdup(given->pattern); - truncated[given->patternlen - 2] = 0; + truncated = dup_and_filter_pattern(given->pattern); translated = xmalloc(sizeof(struct pattern_entry)); translated->pattern = truncated; @@ -737,7 +766,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern translated = xmalloc(sizeof(struct pattern_entry)); - translated->pattern = xstrdup(given->pattern); + translated->pattern = dup_and_filter_pattern(given->pattern); translated->patternlen = given->patternlen; hashmap_entry_init(&translated->ent, ignore_case ? diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index c732abeacd..9ea700896d 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -378,13 +378,28 @@ test_expect_success 'pattern-checks: contained glob characters' ' done ' -test_expect_success 'pattern-checks: escaped "*"' ' - cat >repo/.git/info/sparse-checkout <<-\EOF && +test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' + git clone repo escaped && + TREEOID=$(git -C escaped rev-parse HEAD:folder1) && + NEWTREE=$(git -C escaped mktree <<-EOF + $(git -C escaped ls-tree HEAD) + 040000 tree $TREEOID zbad\\dir + 040000 tree $TREEOID zdoes*exist + EOF + ) && + COMMIT=$(git -C escaped commit-tree $NEWTREE -p HEAD) && + git -C escaped reset --hard $COMMIT && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + git -C escaped sparse-checkout init --cone && + cat >escaped/.git/info/sparse-checkout <<-\EOF && /* !/*/ - /does\*not\*exist/ + /zbad\\dir/ + !/zbad\\dir/*/ + /zdoes\*not\*exist/ + /zdoes\*exist/ EOF - check_read_tree_errors repo "a" "" + check_read_tree_errors escaped "a zbad\\dir zdoes*exist" ' test_done From d585f0e7992ea7f025a5a91f46f2baa9e88f19f6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:10 +0000 Subject: [PATCH 10/19] sparse-checkout: write escaped patterns in cone mode If a user somehow creates a directory with an asterisk (*) or backslash (\), then the "git sparse-checkout set" command will struggle to provide the correct pattern in the sparse-checkout file. When not in cone mode, the provided pattern is written directly into the sparse-checkout file. However, in cone mode we expect a list of paths to directories and then we convert those into patterns. However, there is some care needed for the timing of these escapes. The in-memory pattern list is used to update the working directory before writing the patterns to disk. Thus, we need the command to have the unescaped names in the hashsets for the cone comparisons, then escape the patterns later. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 23 +++++++++++++++++++++-- t/t1091-sparse-checkout-builtin.sh | 10 ++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 3cee8ab46e..cc86b8a014 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -13,6 +13,7 @@ #include "resolve-undo.h" #include "unpack-trees.h" #include "wt-status.h" +#include "quote.h" static const char *empty_base = ""; @@ -140,6 +141,22 @@ static int update_working_directory(struct pattern_list *pl) return result; } +static char *escaped_pattern(char *pattern) +{ + char *p = pattern; + struct strbuf final = STRBUF_INIT; + + while (*p) { + if (*p == '*' || *p == '\\') + strbuf_addch(&final, '\\'); + + strbuf_addch(&final, *p); + p++; + } + + return strbuf_detach(&final, NULL); +} + static void write_cone_to_file(FILE *fp, struct pattern_list *pl) { int i; @@ -164,10 +181,11 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) fprintf(fp, "/*\n!/*/\n"); for (i = 0; i < sl.nr; i++) { - char *pattern = sl.items[i].string; + char *pattern = escaped_pattern(sl.items[i].string); if (strlen(pattern)) fprintf(fp, "%s/\n!%s/*/\n", pattern, pattern); + free(pattern); } string_list_clear(&sl, 0); @@ -185,8 +203,9 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) string_list_remove_duplicates(&sl, 0); for (i = 0; i < sl.nr; i++) { - char *pattern = sl.items[i].string; + char *pattern = escaped_pattern(sl.items[i].string); fprintf(fp, "%s/\n", pattern); + free(pattern); } } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 9ea700896d..fb8718e64a 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -309,6 +309,9 @@ check_read_tree_errors () { REPO=$1 FILES=$2 ERRORS=$3 + git -C $REPO -c core.sparseCheckoutCone=false read-tree -mu HEAD 2>err && + test_must_be_empty err && + check_files $REPO "$FILES" && git -C $REPO read-tree -mu HEAD 2>err && if test -z "$ERRORS" then @@ -391,14 +394,17 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' git -C escaped reset --hard $COMMIT && check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && git -C escaped sparse-checkout init --cone && - cat >escaped/.git/info/sparse-checkout <<-\EOF && + git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" && + cat >expect <<-\EOF && /* !/*/ /zbad\\dir/ !/zbad\\dir/*/ - /zdoes\*not\*exist/ + /zbad\\dir/bogus/ /zdoes\*exist/ + /zdoes\*not\*exist/ EOF + test_cmp expect escaped/.git/info/sparse-checkout && check_read_tree_errors escaped "a zbad\\dir zdoes*exist" ' From bd64de42de28e5cdda7765d5de1c3ed34d4898cb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:11 +0000 Subject: [PATCH 11/19] sparse-checkout: unquote C-style strings over --stdin If a user somehow creates a directory with an asterisk (*) or backslash (\), then the "git sparse-checkout set" command will struggle to provide the correct pattern in the sparse-checkout file. When not in cone mode, the provided pattern is written directly into the sparse-checkout file. However, in cone mode we expect a list of paths to directories and then we convert those into patterns. Even more specifically, the goal is to always allow the following from the root of a repo: git ls-tree --name-only -d HEAD | git sparse-checkout set --stdin The ls-tree command provides directory names with an unescaped asterisk. It also quotes the directories that contain an escaped backslash. We must remove these quotes, then keep the escaped backslashes. Use unquote_c_style() when parsing lines from stdin. Command-line arguments will be parsed as-is, assuming the user can do the correct level of escaping from their environment to match the exact directory names. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 15 ++++++++++++++- t/t1091-sparse-checkout-builtin.sh | 14 +++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index cc86b8a014..6083aa10f2 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -442,8 +442,21 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix) pl.use_cone_patterns = 1; if (set_opts.use_stdin) { - while (!strbuf_getline(&line, stdin)) + struct strbuf unquoted = STRBUF_INIT; + while (!strbuf_getline(&line, stdin)) { + if (line.buf[0] == '"') { + strbuf_reset(&unquoted); + if (unquote_c_style(&unquoted, line.buf, NULL)) + die(_("unable to unquote C-style string '%s'"), + line.buf); + + strbuf_swap(&unquoted, &line); + } + strbuf_to_cone_pattern(&line, &pl); + } + + strbuf_release(&unquoted); } else { for (i = 0; i < argc; i++) { strbuf_setlen(&line, 0); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index fb8718e64a..a46a310740 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -405,7 +405,19 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zdoes\*not\*exist/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_read_tree_errors escaped "a zbad\\dir zdoes*exist" + check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && + git -C escaped ls-tree -d --name-only HEAD | git -C escaped sparse-checkout set --stdin && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + /folder1/ + /folder2/ + /zbad\\dir/ + /zdoes\*exist/ + EOF + test_cmp expect escaped/.git/info/sparse-checkout && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" ' test_done From e55682ea2640dd3aa002a2657c32bdd1d85b44e9 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:12 +0000 Subject: [PATCH 12/19] sparse-checkout: use C-style quotes in 'list' subcommand When in cone mode, the 'git sparse-checkout list' subcommand lists the directories included in the sparse cone. When these directories contain odd characters, such as a backslash, then we need to use C-style quotes similar to 'git ls-tree'. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 6 ++++-- t/t1091-sparse-checkout-builtin.sh | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 6083aa10f2..facdb6bda7 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -78,8 +78,10 @@ static int sparse_checkout_list(int argc, const char **argv) string_list_sort(&sl); - for (i = 0; i < sl.nr; i++) - printf("%s\n", sl.items[i].string); + for (i = 0; i < sl.nr; i++) { + quote_c_style(sl.items[i].string, NULL, stdout, 0); + printf("\n"); + } return 0; } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index a46a310740..545e8d5ebe 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -406,7 +406,8 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' EOF test_cmp expect escaped/.git/info/sparse-checkout && check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && - git -C escaped ls-tree -d --name-only HEAD | git -C escaped sparse-checkout set --stdin && + git -C escaped ls-tree -d --name-only HEAD >list-expect && + git -C escaped sparse-checkout set --stdin expect <<-\EOF && /* !/*/ @@ -417,7 +418,9 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zdoes\*exist/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + git -C escaped sparse-checkout list >list-actual && + test_cmp list-expect list-actual ' test_done From e53ffe2704d7e10690f4382e46c1411a482531f1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:13 +0000 Subject: [PATCH 13/19] sparse-checkout: escape all glob characters on write The sparse-checkout patterns allow special globs according to fnmatch(3). When writing cone-mode patterns for paths containing these characters, they must be escaped. Use is_glob_special() to check which characters must be escaped this way, and add a path to the tests that contains all glob characters at once. Note that ']' is not special, since the initial bracket '[' is escaped. Reported-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 2 +- t/t1091-sparse-checkout-builtin.sh | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index facdb6bda7..7aeb384362 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -149,7 +149,7 @@ static char *escaped_pattern(char *pattern) struct strbuf final = STRBUF_INIT; while (*p) { - if (*p == '*' || *p == '\\') + if (is_glob_special(*p)) strbuf_addch(&final, '\\'); strbuf_addch(&final, *p); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 545e8d5ebe..37e9304ef3 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -381,20 +381,21 @@ test_expect_success 'pattern-checks: contained glob characters' ' done ' -test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' +test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' git clone repo escaped && TREEOID=$(git -C escaped rev-parse HEAD:folder1) && NEWTREE=$(git -C escaped mktree <<-EOF $(git -C escaped ls-tree HEAD) 040000 tree $TREEOID zbad\\dir 040000 tree $TREEOID zdoes*exist + 040000 tree $TREEOID zglob[!a]? EOF ) && COMMIT=$(git -C escaped commit-tree $NEWTREE -p HEAD) && git -C escaped reset --hard $COMMIT && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? && git -C escaped sparse-checkout init --cone && - git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" && + git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" "zglob[!a]?" && cat >expect <<-\EOF && /* !/*/ @@ -403,9 +404,10 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zbad\\dir/bogus/ /zdoes\*exist/ /zdoes\*not\*exist/ + /zglob\[!a]\?/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && + check_read_tree_errors escaped "a zbad\\dir zdoes*exist zglob[!a]?" && git -C escaped ls-tree -d --name-only HEAD >list-expect && git -C escaped sparse-checkout set --stdin expect <<-\EOF && @@ -416,9 +418,10 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /folder2/ /zbad\\dir/ /zdoes\*exist/ + /zglob\[!a]\?/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? && git -C escaped sparse-checkout list >list-actual && test_cmp list-expect list-actual ' From d2e65f4c9056be72ff8a1f39245c5e1b27d556b2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:14 +0000 Subject: [PATCH 14/19] sparse-checkout: improve docs around 'set' in cone mode The existing documentation does not clarify how the 'set' subcommand changes when core.sparseCheckoutCone is enabled. Correct this by changing some language around the "A/B/C" example. Also include a description of the input format matching the output of 'git ls-tree --name-only'. Helped-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-sparse-checkout.txt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 4834fb434d..0914619881 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -50,6 +50,14 @@ To avoid interfering with other worktrees, it first enables the + When the `--stdin` option is provided, the patterns are read from standard in as a newline-delimited list instead of from the arguments. ++ +When `core.sparseCheckoutCone` is enabled, the input list is considered a +list of directories instead of sparse-checkout patterns. The command writes +patterns to the sparse-checkout file to include all files contained in those +directories (recursively) as well as files that are siblings of ancestor +directories. The input format matches the output of `git ls-tree --name-only`. +This includes interpreting pathnames that begin with a double quote (") as +C-style quoted strings. 'disable':: Disable the `core.sparseCheckout` config setting, and restore the @@ -128,9 +136,12 @@ the following patterns: ---------------- This says "include everything in root, but nothing two levels below root." -If we then add the folder `A/B/C` as a recursive pattern, the folders `A` and -`A/B` are added as parent patterns. The resulting sparse-checkout file is -now + +When in cone mode, the `git sparse-checkout set` subcommand takes a list of +directories instead of a list of sparse-checkout patterns. In this mode, +the command `git sparse-checkout set A/B/C` sets the directory `A/B/C` as +a recursive pattern, the directories `A` and `A/B` are added as parent +patterns. The resulting sparse-checkout file is now ---------------- /* From f998a3f1e588d73ed7285cb14ac4839f63f6dc82 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:15 +0000 Subject: [PATCH 15/19] sparse-checkout: fix cone mode behavior mismatch The intention of the special "cone mode" in the sparse-checkout feature is to always match the same patterns that are matched by the same sparse-checkout file as when cone mode is disabled. When a file path is given to "git sparse-checkout set" in cone mode, then the cone mode improperly matches the file as a recursive path. When setting the skip-worktree bits, files were not expecting the MATCHED_RECURSIVE response, and hence these were left out of the matched cone. Fix this bug by checking for MATCHED_RECURSIVE in addition to MATCHED and add a test that prevents regression. Reported-by: Finn Bryant Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1091-sparse-checkout-builtin.sh | 12 ++++++++++++ unpack-trees.c | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 37e9304ef3..7d982096fb 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -305,6 +305,18 @@ test_expect_success 'different sparse-checkouts with worktrees' ' check_files worktree a deep ' +test_expect_success 'set using filename keeps file on-disk' ' + git -C repo sparse-checkout set a deep && + cat >expect <<-\EOF && + /* + !/*/ + /a/ + /deep/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep +' + check_read_tree_errors () { REPO=$1 FILES=$2 diff --git a/unpack-trees.c b/unpack-trees.c index 3789a22cf0..78425ce74b 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1416,7 +1416,7 @@ static int clear_ce_flags_1(struct index_state *istate, name, &dtype, pl, istate); if (ret == UNDECIDED) ret = default_match; - if (ret == MATCHED) + if (ret == MATCHED || ret == MATCHED_RECURSIVE) ce->ce_flags &= ~clear_mask; cache++; progress_nr++; From 6fb705abcb6044f07954b486d71c05151262b6b6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 11 Feb 2020 15:02:21 +0000 Subject: [PATCH 16/19] sparse-checkout: extract add_patterns_from_input() In anticipation of extending the sparse-checkout builtin with "add" and "remove" subcommands, extract the code that fills a pattern list based on the input values. The input changes depending on the presence of "--stdin" or the value of core.sparseCheckoutCone. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 96 +++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 7aeb384362..41d8aaf9a2 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -412,9 +412,58 @@ static struct sparse_checkout_set_opts { int use_stdin; } set_opts; -static int sparse_checkout_set(int argc, const char **argv, const char *prefix) +static void add_patterns_from_input(struct pattern_list *pl, + int argc, const char **argv) { int i; + if (core_sparse_checkout_cone) { + struct strbuf line = STRBUF_INIT; + + hashmap_init(&pl->recursive_hashmap, pl_hashmap_cmp, NULL, 0); + hashmap_init(&pl->parent_hashmap, pl_hashmap_cmp, NULL, 0); + pl->use_cone_patterns = 1; + + if (set_opts.use_stdin) { + struct strbuf unquoted = STRBUF_INIT; + while (!strbuf_getline(&line, stdin)) { + if (line.buf[0] == '"') { + strbuf_reset(&unquoted); + if (unquote_c_style(&unquoted, line.buf, NULL)) + die(_("unable to unquote C-style string '%s'"), + line.buf); + + strbuf_swap(&unquoted, &line); + } + + strbuf_to_cone_pattern(&line, pl); + } + + strbuf_release(&unquoted); + } else { + for (i = 0; i < argc; i++) { + strbuf_setlen(&line, 0); + strbuf_addstr(&line, argv[i]); + strbuf_to_cone_pattern(&line, pl); + } + } + } else { + if (set_opts.use_stdin) { + struct strbuf line = STRBUF_INIT; + + while (!strbuf_getline(&line, stdin)) { + size_t len; + char *buf = strbuf_detach(&line, &len); + add_pattern(buf, empty_base, 0, pl, 0); + } + } else { + for (i = 0; i < argc; i++) + add_pattern(argv[i], empty_base, 0, pl, 0); + } + } +} + +static int sparse_checkout_set(int argc, const char **argv, const char *prefix) +{ struct pattern_list pl; int result; int changed_config = 0; @@ -436,50 +485,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix) builtin_sparse_checkout_set_usage, PARSE_OPT_KEEP_UNKNOWN); - if (core_sparse_checkout_cone) { - struct strbuf line = STRBUF_INIT; - - hashmap_init(&pl.recursive_hashmap, pl_hashmap_cmp, NULL, 0); - hashmap_init(&pl.parent_hashmap, pl_hashmap_cmp, NULL, 0); - pl.use_cone_patterns = 1; - - if (set_opts.use_stdin) { - struct strbuf unquoted = STRBUF_INIT; - while (!strbuf_getline(&line, stdin)) { - if (line.buf[0] == '"') { - strbuf_reset(&unquoted); - if (unquote_c_style(&unquoted, line.buf, NULL)) - die(_("unable to unquote C-style string '%s'"), - line.buf); - - strbuf_swap(&unquoted, &line); - } - - strbuf_to_cone_pattern(&line, &pl); - } - - strbuf_release(&unquoted); - } else { - for (i = 0; i < argc; i++) { - strbuf_setlen(&line, 0); - strbuf_addstr(&line, argv[i]); - strbuf_to_cone_pattern(&line, &pl); - } - } - } else { - if (set_opts.use_stdin) { - struct strbuf line = STRBUF_INIT; - - while (!strbuf_getline(&line, stdin)) { - size_t len; - char *buf = strbuf_detach(&line, &len); - add_pattern(buf, empty_base, 0, &pl, 0); - } - } else { - for (i = 0; i < argc; i++) - add_pattern(argv[i], empty_base, 0, &pl, 0); - } - } + add_patterns_from_input(&pl, argc, argv); if (!core_apply_sparse_checkout) { set_config(MODE_ALL_PATTERNS); From 4bf0c06c7169da61de489544207a7659ef31029f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 11 Feb 2020 15:02:22 +0000 Subject: [PATCH 17/19] sparse-checkout: extract pattern update from 'set' subcommand In anticipation of adding "add" and "remove" subcommands to the sparse-checkout builtin, extract a modify_pattern_list() method from the sparse_checkout_set() method. This command will read input from the command-line or stdin to construct a set of patterns, then modify the existing sparse-checkout patterns after a successful update of the working directory. Currently, the only way to modify the patterns is to replace all of the patterns. This will be extended in a later update. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 44 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 41d8aaf9a2..03915dd729 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -462,29 +462,17 @@ static void add_patterns_from_input(struct pattern_list *pl, } } -static int sparse_checkout_set(int argc, const char **argv, const char *prefix) +enum modify_type { + REPLACE, +}; + +static int modify_pattern_list(int argc, const char **argv, enum modify_type m) { - struct pattern_list pl; int result; int changed_config = 0; - - static struct option builtin_sparse_checkout_set_options[] = { - OPT_BOOL(0, "stdin", &set_opts.use_stdin, - N_("read patterns from standard in")), - OPT_END(), - }; - - repo_read_index(the_repository); - require_clean_work_tree(the_repository, - N_("set sparse-checkout patterns"), NULL, 1, 0); - + struct pattern_list pl; memset(&pl, 0, sizeof(pl)); - argc = parse_options(argc, argv, prefix, - builtin_sparse_checkout_set_options, - builtin_sparse_checkout_set_usage, - PARSE_OPT_KEEP_UNKNOWN); - add_patterns_from_input(&pl, argc, argv); if (!core_apply_sparse_checkout) { @@ -502,6 +490,26 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix) return result; } +static int sparse_checkout_set(int argc, const char **argv, const char *prefix) +{ + static struct option builtin_sparse_checkout_set_options[] = { + OPT_BOOL(0, "stdin", &set_opts.use_stdin, + N_("read patterns from standard in")), + OPT_END(), + }; + + repo_read_index(the_repository); + require_clean_work_tree(the_repository, + N_("set sparse-checkout patterns"), NULL, 1, 0); + + argc = parse_options(argc, argv, prefix, + builtin_sparse_checkout_set_options, + builtin_sparse_checkout_set_usage, + PARSE_OPT_KEEP_UNKNOWN); + + return modify_pattern_list(argc, argv, REPLACE); +} + static int sparse_checkout_disable(int argc, const char **argv) { struct pattern_list pl; From 2631dc879d59aa08095bc4fb5bc9bcc491a787e9 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 11 Feb 2020 15:02:23 +0000 Subject: [PATCH 18/19] sparse-checkout: create 'add' subcommand When using the sparse-checkout feature, a user may want to incrementally grow their sparse-checkout pattern set. Allow adding patterns using a new 'add' subcommand. This is not much different from the 'set' subcommand, because we still want to allow the '--stdin' option and interpret inputs as directories when in cone mode and patterns otherwise. When in cone mode, we are growing the cone. This may actually reduce the set of patterns when adding directory A when A/B is already a directory in the cone. Test the different cases: siblings, parents, ancestors. When not in cone mode, we can only assume the patterns should be appended to the sparse-checkout file. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-sparse-checkout.txt | 7 +++ builtin/sparse-checkout.c | 72 ++++++++++++++++++++++++--- t/t1091-sparse-checkout-builtin.sh | 59 ++++++++++++++++++++++ 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 0914619881..746f920d71 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -59,6 +59,13 @@ directories. The input format matches the output of `git ls-tree --name-only`. This includes interpreting pathnames that begin with a double quote (") as C-style quoted strings. +'add':: + Update the sparse-checkout file to include additional patterns. + By default, these patterns are read from the command-line arguments, + but they can be read from stdin using the `--stdin` option. When + `core.sparseCheckoutCone` is enabled, the given patterns are interpreted + as directory names as in the 'set' subcommand. + 'disable':: Disable the `core.sparseCheckout` config setting, and restore the working directory to include all files. Leaves the sparse-checkout diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 03915dd729..af9e3e5123 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -18,7 +18,7 @@ static const char *empty_base = ""; static char const * const builtin_sparse_checkout_usage[] = { - N_("git sparse-checkout (init|list|set|disable) "), + N_("git sparse-checkout (init|list|set|add|disable) "), NULL }; @@ -404,7 +404,7 @@ static void strbuf_to_cone_pattern(struct strbuf *line, struct pattern_list *pl) } static char const * const builtin_sparse_checkout_set_usage[] = { - N_("git sparse-checkout set (--stdin | )"), + N_("git sparse-checkout (set|add) (--stdin | )"), NULL }; @@ -464,8 +464,54 @@ static void add_patterns_from_input(struct pattern_list *pl, enum modify_type { REPLACE, + ADD, }; +static void add_patterns_cone_mode(int argc, const char **argv, + struct pattern_list *pl) +{ + struct strbuf buffer = STRBUF_INIT; + struct pattern_entry *pe; + struct hashmap_iter iter; + struct pattern_list existing; + char *sparse_filename = get_sparse_checkout_filename(); + + add_patterns_from_input(pl, argc, argv); + + memset(&existing, 0, sizeof(existing)); + existing.use_cone_patterns = core_sparse_checkout_cone; + + if (add_patterns_from_file_to_list(sparse_filename, "", 0, + &existing, NULL)) + die(_("unable to load existing sparse-checkout patterns")); + free(sparse_filename); + + hashmap_for_each_entry(&existing.recursive_hashmap, &iter, pe, ent) { + if (!hashmap_contains_parent(&pl->recursive_hashmap, + pe->pattern, &buffer) || + !hashmap_contains_parent(&pl->parent_hashmap, + pe->pattern, &buffer)) { + strbuf_reset(&buffer); + strbuf_addstr(&buffer, pe->pattern); + insert_recursive_pattern(pl, &buffer); + } + } + + clear_pattern_list(&existing); + strbuf_release(&buffer); +} + +static void add_patterns_literal(int argc, const char **argv, + struct pattern_list *pl) +{ + char *sparse_filename = get_sparse_checkout_filename(); + if (add_patterns_from_file_to_list(sparse_filename, "", 0, + pl, NULL)) + die(_("unable to load existing sparse-checkout patterns")); + free(sparse_filename); + add_patterns_from_input(pl, argc, argv); +} + static int modify_pattern_list(int argc, const char **argv, enum modify_type m) { int result; @@ -473,7 +519,18 @@ static int modify_pattern_list(int argc, const char **argv, enum modify_type m) struct pattern_list pl; memset(&pl, 0, sizeof(pl)); - add_patterns_from_input(&pl, argc, argv); + switch (m) { + case ADD: + if (core_sparse_checkout_cone) + add_patterns_cone_mode(argc, argv, &pl); + else + add_patterns_literal(argc, argv, &pl); + break; + + case REPLACE: + add_patterns_from_input(&pl, argc, argv); + break; + } if (!core_apply_sparse_checkout) { set_config(MODE_ALL_PATTERNS); @@ -490,7 +547,8 @@ static int modify_pattern_list(int argc, const char **argv, enum modify_type m) return result; } -static int sparse_checkout_set(int argc, const char **argv, const char *prefix) +static int sparse_checkout_set(int argc, const char **argv, const char *prefix, + enum modify_type m) { static struct option builtin_sparse_checkout_set_options[] = { OPT_BOOL(0, "stdin", &set_opts.use_stdin, @@ -507,7 +565,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix) builtin_sparse_checkout_set_usage, PARSE_OPT_KEEP_UNKNOWN); - return modify_pattern_list(argc, argv, REPLACE); + return modify_pattern_list(argc, argv, m); } static int sparse_checkout_disable(int argc, const char **argv) @@ -558,7 +616,9 @@ int cmd_sparse_checkout(int argc, const char **argv, const char *prefix) if (!strcmp(argv[0], "init")) return sparse_checkout_init(argc, argv); if (!strcmp(argv[0], "set")) - return sparse_checkout_set(argc, argv, prefix); + return sparse_checkout_set(argc, argv, prefix, REPLACE); + if (!strcmp(argv[0], "add")) + return sparse_checkout_set(argc, argv, prefix, ADD); if (!strcmp(argv[0], "disable")) return sparse_checkout_disable(argc, argv); } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 7d982096fb..f9265de5e8 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -141,6 +141,21 @@ test_expect_success 'set sparse-checkout using --stdin' ' check_files repo "a folder1 folder2" ' +test_expect_success 'add to sparse-checkout' ' + cat repo/.git/info/sparse-checkout >expect && + cat >add <<-\EOF && + pattern1 + /folder1/ + pattern2 + EOF + cat add >>expect && + git -C repo sparse-checkout add --stdin actual && + test_cmp expect actual && + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo "a folder1 folder2" +' + test_expect_success 'cone mode: match patterns' ' git -C repo config --worktree core.sparseCheckoutCone true && rm -rf repo/a repo/folder1 repo/folder2 && @@ -219,8 +234,52 @@ test_expect_success 'cone mode: set with nested folders' ' test_cmp repo/.git/info/sparse-checkout expect ' +test_expect_success 'cone mode: add independent path' ' + git -C repo sparse-checkout set deep/deeper1 && + git -C repo sparse-checkout add folder1 && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + /folder1/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep folder1 +' + +test_expect_success 'cone mode: add sibling path' ' + git -C repo sparse-checkout set deep/deeper1 && + git -C repo sparse-checkout add deep/deeper2 && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + /deep/deeper2/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep +' + +test_expect_success 'cone mode: add parent path' ' + git -C repo sparse-checkout set deep/deeper1 folder1 && + git -C repo sparse-checkout add deep && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + /folder1/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep folder1 +' + test_expect_success 'revert to old sparse-checkout on bad update' ' test_when_finished git -C repo reset --hard && + git -C repo sparse-checkout set deep && echo update >repo/deep/deeper2/a && cp repo/.git/info/sparse-checkout expect && test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && From ef07659926f64d70e8cb41025c3d7456eecb962e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 11 Feb 2020 15:02:24 +0000 Subject: [PATCH 19/19] sparse-checkout: work with Windows paths When using Windows, a user may run 'git sparse-checkout set A\B\C' to add the Unix-style path A/B/C to their sparse-checkout patterns. Normalizing the input path converts the backslashes to slashes before we add the string 'A/B/C' to the recursive hashset. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/sparse-checkout.c | 3 +++ t/t1091-sparse-checkout-builtin.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index af9e3e5123..3e314e3358 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -394,6 +394,9 @@ static void strbuf_to_cone_pattern(struct strbuf *line, struct pattern_list *pl) strbuf_trim_trailing_dir_sep(line); + if (strbuf_normalize_path(line)) + die(_("could not normalize path %s"), line->buf); + if (!line->len) return; diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index f9265de5e8..c35cbdef45 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -497,4 +497,18 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' test_cmp list-expect list-actual ' +test_expect_success MINGW 'cone mode replaces backslashes with slashes' ' + git -C repo sparse-checkout set deep\\deeper1 && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep && + check_files repo/deep a deeper1 +' + test_done