Merge branch 'ds/sparse-checkout-harden'

Some rough edges in the sparse-checkout feature, especially around
the cone mode, have been cleaned up.

* ds/sparse-checkout-harden:
  sparse-checkout: fix cone mode behavior mismatch
  sparse-checkout: improve docs around 'set' in cone mode
  sparse-checkout: escape all glob characters on write
  sparse-checkout: use C-style quotes in 'list' subcommand
  sparse-checkout: unquote C-style strings over --stdin
  sparse-checkout: write escaped patterns in cone mode
  sparse-checkout: properly match escaped characters
  sparse-checkout: warn on globs in cone patterns
  sparse-checkout: detect short patterns
  sparse-checkout: cone mode does not recognize "**"
  sparse-checkout: fix documentation typo for core.sparseCheckoutCone
  clone: fix --sparse option with URLs
  sparse-checkout: create leading directories
  t1091: improve here-docs
  t1091: use check_files to reduce boilerplate
This commit is contained in:
Junio C Hamano 2020-02-14 12:54:22 -08:00
Родитель 4a77434bc8 f998a3f1e5
Коммит 433b8aac2e
6 изменённых файлов: 346 добавлений и 156 удалений

Просмотреть файл

@ -54,6 +54,14 @@ patterns (see 'CONE PATTERN SET' below).
+
When the `--stdin` option is provided, the patterns are read from
standard in as a newline-delimited list instead of from the arguments.
+
When `core.sparseCheckoutCone` is enabled, the input list is considered a
list of directories instead of sparse-checkout patterns. The command writes
patterns to the sparse-checkout file to include all files contained in those
directories (recursively) as well as files that are siblings of ancestor
directories. The input format matches the output of `git ls-tree --name-only`.
This includes interpreting pathnames that begin with a double quote (") as
C-style quoted strings.
'disable'::
Disable the `core.sparseCheckout` config setting, and restore the
@ -110,7 +118,7 @@ The full pattern set allows for arbitrary pattern matches and complicated
inclusion/exclusion rules. These can result in O(N*M) pattern matches when
updating the index, where N is the number of patterns and M is the number
of paths in the index. To combat this performance issue, a more restricted
pattern set is allowed when `core.spareCheckoutCone` is enabled.
pattern set is allowed when `core.sparseCheckoutCone` is enabled.
The accepted patterns in the cone pattern set are:
@ -132,9 +140,12 @@ the following patterns:
----------------
This says "include everything in root, but nothing two levels below root."
If we then add the folder `A/B/C` as a recursive pattern, the folders `A` and
`A/B` are added as parent patterns. The resulting sparse-checkout file is
now
When in cone mode, the `git sparse-checkout set` subcommand takes a list of
directories instead of a list of sparse-checkout patterns. In this mode,
the command `git sparse-checkout set A/B/C` sets the directory `A/B/C` as
a recursive pattern, the directories `A` and `A/B` are added as parent
patterns. The resulting sparse-checkout file is now
----------------
/*

Просмотреть файл

@ -1129,7 +1129,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
if (option_required_reference.nr || option_optional_reference.nr)
setup_reference();
if (option_sparse_checkout && git_sparse_checkout_init(repo))
if (option_sparse_checkout && git_sparse_checkout_init(dir))
return 1;
remote = remote_get(option_origin);

Просмотреть файл

@ -13,6 +13,7 @@
#include "resolve-undo.h"
#include "unpack-trees.h"
#include "wt-status.h"
#include "quote.h"
static const char *empty_base = "";
@ -77,8 +78,10 @@ static int sparse_checkout_list(int argc, const char **argv)
string_list_sort(&sl);
for (i = 0; i < sl.nr; i++)
printf("%s\n", sl.items[i].string);
for (i = 0; i < sl.nr; i++) {
quote_c_style(sl.items[i].string, NULL, stdout, 0);
printf("\n");
}
return 0;
}
@ -140,6 +143,22 @@ static int update_working_directory(struct pattern_list *pl)
return result;
}
static char *escaped_pattern(char *pattern)
{
char *p = pattern;
struct strbuf final = STRBUF_INIT;
while (*p) {
if (is_glob_special(*p))
strbuf_addch(&final, '\\');
strbuf_addch(&final, *p);
p++;
}
return strbuf_detach(&final, NULL);
}
static void write_cone_to_file(FILE *fp, struct pattern_list *pl)
{
int i;
@ -164,10 +183,11 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl)
fprintf(fp, "/*\n!/*/\n");
for (i = 0; i < sl.nr; i++) {
char *pattern = sl.items[i].string;
char *pattern = escaped_pattern(sl.items[i].string);
if (strlen(pattern))
fprintf(fp, "%s/\n!%s/*/\n", pattern, pattern);
free(pattern);
}
string_list_clear(&sl, 0);
@ -185,8 +205,9 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl)
string_list_remove_duplicates(&sl, 0);
for (i = 0; i < sl.nr; i++) {
char *pattern = sl.items[i].string;
char *pattern = escaped_pattern(sl.items[i].string);
fprintf(fp, "%s/\n", pattern);
free(pattern);
}
}
@ -199,6 +220,10 @@ static int write_patterns_and_update(struct pattern_list *pl)
int result;
sparse_filename = get_sparse_checkout_filename();
if (safe_create_leading_directories(sparse_filename))
die(_("failed to create directory for sparse-checkout file"));
fd = hold_lock_file_for_update(&lk, sparse_filename,
LOCK_DIE_ON_ERROR);
@ -419,8 +444,21 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix)
pl.use_cone_patterns = 1;
if (set_opts.use_stdin) {
while (!strbuf_getline(&line, stdin))
struct strbuf unquoted = STRBUF_INIT;
while (!strbuf_getline(&line, stdin)) {
if (line.buf[0] == '"') {
strbuf_reset(&unquoted);
if (unquote_c_style(&unquoted, line.buf, NULL))
die(_("unable to unquote C-style string '%s'"),
line.buf);
strbuf_swap(&unquoted, &line);
}
strbuf_to_cone_pattern(&line, &pl);
}
strbuf_release(&unquoted);
} else {
for (i = 0; i < argc; i++) {
strbuf_setlen(&line, 0);

79
dir.c
Просмотреть файл

@ -636,11 +636,42 @@ int pl_hashmap_cmp(const void *unused_cmp_data,
return strncmp(ee1->pattern, ee2->pattern, min_len);
}
static char *dup_and_filter_pattern(const char *pattern)
{
char *set, *read;
size_t count = 0;
char *result = xstrdup(pattern);
set = result;
read = result;
while (*read) {
/* skip escape characters (once) */
if (*read == '\\')
read++;
*set = *read;
set++;
read++;
count++;
}
*set = 0;
if (count > 2 &&
*(set - 1) == '*' &&
*(set - 2) == '/')
*(set - 2) = 0;
return result;
}
static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern *given)
{
struct pattern_entry *translated;
char *truncated;
char *data = NULL;
const char *prev, *cur, *next;
if (!pl->use_cone_patterns)
return;
@ -657,17 +688,57 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern
return;
}
if (given->patternlen <= 2 ||
*given->pattern == '*' ||
strstr(given->pattern, "**")) {
/* Not a cone pattern. */
warning(_("unrecognized pattern: '%s'"), given->pattern);
goto clear_hashmaps;
}
prev = given->pattern;
cur = given->pattern + 1;
next = given->pattern + 2;
while (*cur) {
/* Watch for glob characters '*', '\', '[', '?' */
if (!is_glob_special(*cur))
goto increment;
/* But only if *prev != '\\' */
if (*prev == '\\')
goto increment;
/* But allow the initial '\' */
if (*cur == '\\' &&
is_glob_special(*next))
goto increment;
/* But a trailing '/' then '*' is fine */
if (*prev == '/' &&
*cur == '*' &&
*next == 0)
goto increment;
/* Not a cone pattern. */
warning(_("unrecognized pattern: '%s'"), given->pattern);
goto clear_hashmaps;
increment:
prev++;
cur++;
next++;
}
if (given->patternlen > 2 &&
!strcmp(given->pattern + given->patternlen - 2, "/*")) {
if (!(given->flags & PATTERN_FLAG_NEGATIVE)) {
/* Not a cone pattern. */
pl->use_cone_patterns = 0;
warning(_("unrecognized pattern: '%s'"), given->pattern);
goto clear_hashmaps;
}
truncated = xstrdup(given->pattern);
truncated[given->patternlen - 2] = 0;
truncated = dup_and_filter_pattern(given->pattern);
translated = xmalloc(sizeof(struct pattern_entry));
translated->pattern = truncated;
@ -701,7 +772,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern
translated = xmalloc(sizeof(struct pattern_entry));
translated->pattern = xstrdup(given->pattern);
translated->pattern = dup_and_filter_pattern(given->pattern);
translated->patternlen = given->patternlen;
hashmap_entry_init(&translated->ent,
ignore_case ?

Просмотреть файл

@ -12,6 +12,13 @@ list_files() {
(cd "$1" && printf '%s\n' *)
}
check_files() {
list_files "$1" >actual &&
shift &&
printf "%s\n" $@ >expect &&
test_cmp expect actual
}
test_expect_success 'setup' '
git init repo &&
(
@ -39,11 +46,11 @@ test_expect_success 'git sparse-checkout list (empty)' '
test_expect_success 'git sparse-checkout list (populated)' '
test_when_finished rm -f repo/.git/info/sparse-checkout &&
cat >repo/.git/info/sparse-checkout <<-EOF &&
/folder1/*
/deep/
**/a
!*bin*
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/folder1/*
/deep/
**/a
!*bin*
EOF
cp repo/.git/info/sparse-checkout expect &&
git -C repo sparse-checkout list >list &&
@ -52,22 +59,20 @@ test_expect_success 'git sparse-checkout list (populated)' '
test_expect_success 'git sparse-checkout init' '
git -C repo sparse-checkout init &&
cat >expect <<-EOF &&
/*
!/*/
cat >expect <<-\EOF &&
/*
!/*/
EOF
test_cmp expect repo/.git/info/sparse-checkout &&
test_cmp_config -C repo true core.sparsecheckout &&
list_files repo >dir &&
echo a >expect &&
test_cmp expect dir
check_files repo a
'
test_expect_success 'git sparse-checkout list after init' '
git -C repo sparse-checkout list >actual &&
cat >expect <<-EOF &&
/*
!/*/
cat >expect <<-\EOF &&
/*
!/*/
EOF
test_cmp expect actual
'
@ -75,32 +80,24 @@ test_expect_success 'git sparse-checkout list after init' '
test_expect_success 'init with existing sparse-checkout' '
echo "*folder*" >> repo/.git/info/sparse-checkout &&
git -C repo sparse-checkout init &&
cat >expect <<-EOF &&
/*
!/*/
*folder*
cat >expect <<-\EOF &&
/*
!/*/
*folder*
EOF
test_cmp expect repo/.git/info/sparse-checkout &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
folder1
folder2
EOF
test_cmp expect dir
check_files repo a folder1 folder2
'
test_expect_success 'clone --sparse' '
git clone --sparse repo clone &&
git clone --sparse "file://$(pwd)/repo" clone &&
git -C clone sparse-checkout list >actual &&
cat >expect <<-EOF &&
/*
!/*/
cat >expect <<-\EOF &&
/*
!/*/
EOF
test_cmp expect actual &&
list_files clone >dir &&
echo a >expect &&
test_cmp expect dir
check_files clone a
'
test_expect_success 'set enables config' '
@ -119,41 +116,29 @@ test_expect_success 'set enables config' '
test_expect_success 'set sparse-checkout using builtin' '
git -C repo sparse-checkout set "/*" "!/*/" "*folder*" &&
cat >expect <<-EOF &&
/*
!/*/
*folder*
cat >expect <<-\EOF &&
/*
!/*/
*folder*
EOF
git -C repo sparse-checkout list >actual &&
test_cmp expect actual &&
test_cmp expect repo/.git/info/sparse-checkout &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
folder1
folder2
EOF
test_cmp expect dir
check_files repo a folder1 folder2
'
test_expect_success 'set sparse-checkout using --stdin' '
cat >expect <<-EOF &&
/*
!/*/
/folder1/
/folder2/
cat >expect <<-\EOF &&
/*
!/*/
/folder1/
/folder2/
EOF
git -C repo sparse-checkout set --stdin <expect &&
git -C repo sparse-checkout list >actual &&
test_cmp expect actual &&
test_cmp expect repo/.git/info/sparse-checkout &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
folder1
folder2
EOF
test_cmp expect dir
check_files repo "a folder1 folder2"
'
test_expect_success 'cone mode: match patterns' '
@ -162,13 +147,7 @@ test_expect_success 'cone mode: match patterns' '
git -C repo read-tree -mu HEAD 2>err &&
test_i18ngrep ! "disabling cone patterns" err &&
git -C repo reset --hard &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
folder1
folder2
EOF
test_cmp expect dir
check_files repo a folder1 folder2
'
test_expect_success 'cone mode: warn on bad pattern' '
@ -185,14 +164,7 @@ test_expect_success 'sparse-checkout disable' '
test_path_is_file repo/.git/info/sparse-checkout &&
git -C repo config --list >config &&
test_must_fail git config core.sparseCheckout &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
deep
folder1
folder2
EOF
test_cmp expect dir
check_files repo a deep folder1 folder2
'
test_expect_success 'cone mode: init and set' '
@ -204,52 +176,31 @@ test_expect_success 'cone mode: init and set' '
test_cmp expect dir &&
git -C repo sparse-checkout set deep/deeper1/deepest/ 2>err &&
test_must_be_empty err &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
deep
EOF
test_cmp expect dir &&
list_files repo/deep >dir &&
cat >expect <<-EOF &&
a
deeper1
EOF
test_cmp expect dir &&
list_files repo/deep/deeper1 >dir &&
cat >expect <<-EOF &&
a
deepest
EOF
test_cmp expect dir &&
cat >expect <<-EOF &&
/*
!/*/
/deep/
!/deep/*/
/deep/deeper1/
!/deep/deeper1/*/
/deep/deeper1/deepest/
check_files repo a deep &&
check_files repo/deep a deeper1 &&
check_files repo/deep/deeper1 a deepest &&
cat >expect <<-\EOF &&
/*
!/*/
/deep/
!/deep/*/
/deep/deeper1/
!/deep/deeper1/*/
/deep/deeper1/deepest/
EOF
test_cmp expect repo/.git/info/sparse-checkout &&
git -C repo sparse-checkout set --stdin 2>err <<-EOF &&
folder1
folder2
git -C repo sparse-checkout set --stdin 2>err <<-\EOF &&
folder1
folder2
EOF
test_must_be_empty err &&
cat >expect <<-EOF &&
a
folder1
folder2
EOF
list_files repo >dir &&
test_cmp expect dir
check_files repo a folder1 folder2
'
test_expect_success 'cone mode: list' '
cat >expect <<-EOF &&
folder1
folder2
cat >expect <<-\EOF &&
folder1
folder2
EOF
git -C repo sparse-checkout set --stdin <expect &&
git -C repo sparse-checkout list >actual 2>err &&
@ -260,10 +211,10 @@ test_expect_success 'cone mode: list' '
test_expect_success 'cone mode: set with nested folders' '
git -C repo sparse-checkout set deep deep/deeper1/deepest 2>err &&
test_line_count = 0 err &&
cat >expect <<-EOF &&
/*
!/*/
/deep/
cat >expect <<-\EOF &&
/*
!/*/
/deep/
EOF
test_cmp repo/.git/info/sparse-checkout expect
'
@ -275,13 +226,7 @@ test_expect_success 'revert to old sparse-checkout on bad update' '
test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err &&
test_i18ngrep "cannot set sparse-checkout patterns" err &&
test_cmp repo/.git/info/sparse-checkout expect &&
list_files repo/deep >dir &&
cat >expect <<-EOF &&
a
deeper1
deeper2
EOF
test_cmp dir expect
check_files repo/deep a deeper1 deeper2
'
test_expect_success 'revert to old sparse-checkout on empty update' '
@ -326,18 +271,13 @@ test_expect_success 'sparse-checkout (init|set|disable) fails with dirty status'
test_expect_success 'cone mode: set with core.ignoreCase=true' '
git -C repo sparse-checkout init --cone &&
git -C repo -c core.ignoreCase=true sparse-checkout set folder1 &&
cat >expect <<-EOF &&
/*
!/*/
/folder1/
cat >expect <<-\EOF &&
/*
!/*/
/folder1/
EOF
test_cmp expect repo/.git/info/sparse-checkout &&
list_files repo >dir &&
cat >expect <<-EOF &&
a
folder1
EOF
test_cmp expect dir
check_files repo a folder1
'
test_expect_success 'interaction with submodules' '
@ -351,21 +291,151 @@ test_expect_success 'interaction with submodules' '
git sparse-checkout init --cone &&
git sparse-checkout set folder1
) &&
list_files super >dir &&
check_files super a folder1 modules &&
check_files super/modules/child a deep folder1 folder2
'
test_expect_success 'different sparse-checkouts with worktrees' '
git -C repo worktree add --detach ../worktree &&
check_files worktree "a deep folder1 folder2" &&
git -C worktree sparse-checkout init --cone &&
git -C repo sparse-checkout set folder1 &&
git -C worktree sparse-checkout set deep/deeper1 &&
check_files repo a folder1 &&
check_files worktree a deep
'
test_expect_success 'set using filename keeps file on-disk' '
git -C repo sparse-checkout set a deep &&
cat >expect <<-\EOF &&
a
folder1
modules
/*
!/*/
/a/
/deep/
EOF
test_cmp expect dir &&
list_files super/modules/child >dir &&
test_cmp expect repo/.git/info/sparse-checkout &&
check_files repo a deep
'
check_read_tree_errors () {
REPO=$1
FILES=$2
ERRORS=$3
git -C $REPO -c core.sparseCheckoutCone=false read-tree -mu HEAD 2>err &&
test_must_be_empty err &&
check_files $REPO "$FILES" &&
git -C $REPO read-tree -mu HEAD 2>err &&
if test -z "$ERRORS"
then
test_must_be_empty err
else
test_i18ngrep "$ERRORS" err
fi &&
check_files $REPO $FILES
}
test_expect_success 'pattern-checks: /A/**' '
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/*
!/*/
/folder1/**
EOF
check_read_tree_errors repo "a folder1" "disabling cone pattern matching"
'
test_expect_success 'pattern-checks: /A/**/B/' '
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/*
!/*/
/deep/**/deepest
EOF
check_read_tree_errors repo "a deep" "disabling cone pattern matching" &&
check_files repo/deep "deeper1" &&
check_files repo/deep/deeper1 "deepest"
'
test_expect_success 'pattern-checks: too short' '
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/*
!/*/
/a
EOF
check_read_tree_errors repo "a" "disabling cone pattern matching"
'
test_expect_success 'pattern-checks: trailing "*"' '
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/*
!/*/
/a*
EOF
check_read_tree_errors repo "a" "disabling cone pattern matching"
'
test_expect_success 'pattern-checks: starting "*"' '
cat >repo/.git/info/sparse-checkout <<-\EOF &&
/*
!/*/
*eep/
EOF
check_read_tree_errors repo "a deep" "disabling cone pattern matching"
'
test_expect_success 'pattern-checks: contained glob characters' '
for c in "[a]" "\\" "?" "*"
do
cat >repo/.git/info/sparse-checkout <<-EOF &&
/*
!/*/
something$c-else/
EOF
check_read_tree_errors repo "a" "disabling cone pattern matching"
done
'
test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' '
git clone repo escaped &&
TREEOID=$(git -C escaped rev-parse HEAD:folder1) &&
NEWTREE=$(git -C escaped mktree <<-EOF
$(git -C escaped ls-tree HEAD)
040000 tree $TREEOID zbad\\dir
040000 tree $TREEOID zdoes*exist
040000 tree $TREEOID zglob[!a]?
EOF
) &&
COMMIT=$(git -C escaped commit-tree $NEWTREE -p HEAD) &&
git -C escaped reset --hard $COMMIT &&
check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? &&
git -C escaped sparse-checkout init --cone &&
git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" "zglob[!a]?" &&
cat >expect <<-\EOF &&
a
deep
folder1
folder2
/*
!/*/
/zbad\\dir/
!/zbad\\dir/*/
/zbad\\dir/bogus/
/zdoes\*exist/
/zdoes\*not\*exist/
/zglob\[!a]\?/
EOF
test_cmp expect dir
test_cmp expect escaped/.git/info/sparse-checkout &&
check_read_tree_errors escaped "a zbad\\dir zdoes*exist zglob[!a]?" &&
git -C escaped ls-tree -d --name-only HEAD >list-expect &&
git -C escaped sparse-checkout set --stdin <list-expect &&
cat >expect <<-\EOF &&
/*
!/*/
/deep/
/folder1/
/folder2/
/zbad\\dir/
/zdoes\*exist/
/zglob\[!a]\?/
EOF
test_cmp expect escaped/.git/info/sparse-checkout &&
check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? &&
git -C escaped sparse-checkout list >list-actual &&
test_cmp list-expect list-actual
'
test_done

Просмотреть файл

@ -1420,7 +1420,7 @@ static int clear_ce_flags_1(struct index_state *istate,
name, &dtype, pl, istate);
if (ret == UNDECIDED)
ret = default_match;
if (ret == MATCHED)
if (ret == MATCHED || ret == MATCHED_RECURSIVE)
ce->ce_flags &= ~clear_mask;
cache++;
progress_nr++;