From 328a43518244b970e1765eca78060bfeb265a584 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 24 Oct 2018 08:56:10 -0700 Subject: [PATCH 1/3] repack: point out a bug handling stale shallow info A `git fetch --prune` can turn previously-reachable objects unreachable, even commits that are in the `shallow` list. A subsequent `git repack -ad` will then unceremoniously drop those unreachable commits, and the `shallow` list will become stale. This means that when we try to fetch with a larger `--depth` the next time, we may end up with: fatal: error in object: unshallow Reported by Alejandro Pauly. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- t/t5537-fetch-shallow.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 7045685e2d..686fdc9280 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -186,6 +186,33 @@ EOF test_cmp expect actual ' +test_expect_failure '.git/shallow is edited by repack' ' + git init shallow-server && + test_commit -C shallow-server A && + test_commit -C shallow-server B && + git -C shallow-server checkout -b branch && + test_commit -C shallow-server C && + test_commit -C shallow-server E && + test_commit -C shallow-server D && + d="$(git -C shallow-server rev-parse --verify D^0)" && + git -C shallow-server checkout master && + + git clone --depth=1 --no-tags --no-single-branch \ + "file://$PWD/shallow-server" shallow-client && + + : now remove the branch and fetch with prune && + git -C shallow-server branch -D branch && + git -C shallow-client fetch --prune --depth=1 \ + origin "+refs/heads/*:refs/remotes/origin/*" && + git -C shallow-client repack -adfl && + test_must_fail git -C shallow-client rev-parse --verify $d^0 && + ! grep $d shallow-client/.git/shallow && + + git -C shallow-server branch branch-orig $d && + git -C shallow-client fetch --prune --depth=2 \ + origin "+refs/heads/*:refs/remotes/origin/*" +' + . "$TEST_DIRECTORY"/lib-httpd.sh start_httpd From 2588f6ed8bd4e31c1ea1ae35f9f668452b46f1ef Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 24 Oct 2018 08:56:12 -0700 Subject: [PATCH 2/3] shallow: offer to prune only non-existing entries The `prune_shallow()` function wants a full reachability check to be completed before it goes to work, to ensure that all unreachable entries are removed from the shallow file. However, in the upcoming patch we do not even want to go that far. We really only need to remove entries corresponding to pruned commits, i.e. to commits that no longer exist. Let's support that use case. Rather than extending the signature of `prune_shallow()` to accept another Boolean, let's turn it into a bit field and declare constants, for readability. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- builtin/prune.c | 2 +- commit.h | 4 +++- shallow.c | 23 +++++++++++++++++------ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/builtin/prune.c b/builtin/prune.c index 4916a4daa2..b29ce4abbc 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -161,7 +161,7 @@ int cmd_prune(int argc, const char **argv, const char *prefix) free(s); if (is_repository_shallow(the_repository)) - prune_shallow(show_only); + prune_shallow(show_only ? PRUNE_SHOW_ONLY : 0); return 0; } diff --git a/commit.h b/commit.h index da0db36eba..861a159314 100644 --- a/commit.h +++ b/commit.h @@ -255,7 +255,9 @@ extern void assign_shallow_commits_to_refs(struct shallow_info *info, uint32_t **used, int *ref_status); extern int delayed_reachability_test(struct shallow_info *si, int c); -extern void prune_shallow(int show_only); +#define PRUNE_SHOW_ONLY 1 +#define PRUNE_QUICK 2 +extern void prune_shallow(unsigned options); extern struct trace_key trace_shallow; int is_descendant_of(struct commit *, struct commit_list *); diff --git a/shallow.c b/shallow.c index dbe8a2a290..c1b68533ca 100644 --- a/shallow.c +++ b/shallow.c @@ -246,6 +246,7 @@ static void check_shallow_file_for_update(struct repository *r) #define SEEN_ONLY 1 #define VERBOSE 2 +#define QUICK 4 struct write_shallow_data { struct strbuf *out; @@ -260,7 +261,10 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) const char *hex = oid_to_hex(&graft->oid); if (graft->nr_parent != -1) return 0; - if (data->flags & SEEN_ONLY) { + if (data->flags & QUICK) { + if (!has_object_file(&graft->oid)) + return 0; + } else if (data->flags & SEEN_ONLY) { struct commit *c = lookup_commit(the_repository, &graft->oid); if (!c || !(c->object.flags & SEEN)) { if (data->flags & VERBOSE) @@ -370,16 +374,23 @@ void advertise_shallow_grafts(int fd) /* * mark_reachable_objects() should have been run prior to this and all - * reachable commits marked as "SEEN". + * reachable commits marked as "SEEN", except when quick_prune is non-zero, + * in which case lines are excised from the shallow file if they refer to + * commits that do not exist (any longer). */ -void prune_shallow(int show_only) +void prune_shallow(unsigned options) { struct lock_file shallow_lock = LOCK_INIT; struct strbuf sb = STRBUF_INIT; + unsigned flags = SEEN_ONLY; int fd; - if (show_only) { - write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY | VERBOSE); + if (options & PRUNE_QUICK) + flags |= QUICK; + + if (options & PRUNE_SHOW_ONLY) { + flags |= VERBOSE; + write_shallow_commits_1(&sb, 0, NULL, flags); strbuf_release(&sb); return; } @@ -387,7 +398,7 @@ void prune_shallow(int show_only) git_path_shallow(the_repository), LOCK_DIE_ON_ERROR); check_shallow_file_for_update(the_repository); - if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) { + if (write_shallow_commits_1(&sb, 0, NULL, flags)) { if (write_in_full(fd, sb.buf, sb.len) < 0) die_errno("failed to write to %s", get_lock_file_path(&shallow_lock)); From 5dcfbf564c0f10869e568af4e05421f63b44fbbf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 24 Oct 2018 08:56:13 -0700 Subject: [PATCH 3/3] repack -ad: prune the list of shallow commits `git repack` can drop unreachable commits without further warning, making the corresponding entries in `.git/shallow` invalid, which causes serious problems when deepening the branches. One scenario where unreachable commits are dropped by `git repack` is when a `git fetch --prune` (or even a `git fetch` when a ref was force-pushed in the meantime) can make a commit unreachable that was reachable before. Therefore it is not safe to assume that a `git repack -adlf` will keep unreachable commits alone (under the assumption that they had not been packed in the first place, which is an assumption at least some of Git's code seems to make). This is particularly important to keep in mind when looking at the `.git/shallow` file: if any commits listed in that file become unreachable, it is not a problem, but if they go missing, it *is* a problem. One symptom of this problem is that a deepening fetch may now fail with fatal: error in object: unshallow To avoid this problem, let's prune the shallow list in `git repack` when the `-d` option is passed, unless `-A` is passed, too (which would force the now-unreachable objects to be turned into loose objects instead of being deleted). Additionally, we also need to take `--keep-reachable` and `--unpack-unreachable=` into account. Note: an alternative solution discussed during the review of this patch was to teach `git fetch` to simply ignore entries in .git/shallow if the corresponding commits do not exist locally. A quick test, however, revealed that the .git/shallow file is written during a shallow *clone*, in which case the commits do not exist, either, but the "shallow" line *does* need to be sent. Therefore, this approach would be a lot more finicky than the approach presented by the this patch. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- builtin/repack.c | 6 ++++++ t/t5537-fetch-shallow.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/builtin/repack.c b/builtin/repack.c index d5886039cc..3c30e16ec5 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -531,6 +531,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!po_args.quiet && isatty(2)) opts |= PRUNE_PACKED_VERBOSE; prune_packed_objects(opts); + + if (!keep_unreachable && + (!(pack_everything & LOOSEN_UNREACHABLE) || + unpack_unreachable) && + is_repository_shallow(the_repository)) + prune_shallow(PRUNE_QUICK); } if (!no_update_server_info) diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 686fdc9280..6faf17e17a 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -186,7 +186,7 @@ EOF test_cmp expect actual ' -test_expect_failure '.git/shallow is edited by repack' ' +test_expect_success '.git/shallow is edited by repack' ' git init shallow-server && test_commit -C shallow-server A && test_commit -C shallow-server B &&