From 657673f125008da1ef92d2054c9536722c42f73b Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 14 Mar 2014 22:26:21 -0400 Subject: [PATCH 1/2] pack-objects: show progress for reused packfiles When the "--all-progress" option is in effect, pack-objects shows a progress report for the "writing" phase. If the repository has bitmaps and we are reusing a packfile, the user sees no progress update until the whole packfile is sent. Since this is typically the bulk of what is being written, it can look like git hangs during this phase, even though the transfer is proceeding. This generally only happens with "git push" from a repository with bitmaps. We do not use "--all-progress" for fetch (since the result is going to index-pack on the client, which takes care of progress reporting). And for regular repacks to disk, we do not reuse packfiles. We already have the progress meter setup during write_reused_pack; we just need to call display_progress whiel we are writing out the pack. The progress meter is attached to our output descriptor, so it automatically handles the throughput measurements. However, we need to update the object count as we go, since that is what feeds the percentage we show. We aren't actually parsing the packfile as we send it, so we have no idea how many objects we have sent; we only know that at the end of N bytes, we will have sent M objects. So we cheat a little and assume each object is M/N bytes (i.e., the mean of the objects we are sending). While this isn't strictly true, it actually produces a more pleasing progress meter for the user, as it moves smoothly and predictably (and nobody really cares about the object count; they care about the percentage, and the object count is a proxy for that). One alternative would be to actually show two progress meters: one for the reused pack, and one for the rest of the objects. That would more closely reflect the data we have (the first would be measured in bytes, and the second measured in objects). But it would also be more complex and annoying to the user; rather than seeing one progress meter counting up to 100%, they would finish one meter, then start another one at zero. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1fb972f45a..f0474db991 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -708,7 +708,7 @@ static struct object_entry **compute_write_order(void) static off_t write_reused_pack(struct sha1file *f) { unsigned char buffer[8192]; - off_t to_write; + off_t to_write, total; int fd; if (!is_pack_valid(reuse_packfile)) @@ -725,7 +725,7 @@ static off_t write_reused_pack(struct sha1file *f) if (reuse_packfile_offset < 0) reuse_packfile_offset = reuse_packfile->pack_size - 20; - to_write = reuse_packfile_offset - sizeof(struct pack_header); + total = to_write = reuse_packfile_offset - sizeof(struct pack_header); while (to_write) { int read_pack = xread(fd, buffer, sizeof(buffer)); @@ -738,10 +738,23 @@ static off_t write_reused_pack(struct sha1file *f) sha1write(f, buffer, read_pack); to_write -= read_pack; + + /* + * We don't know the actual number of objects written, + * only how many bytes written, how many bytes total, and + * how many objects total. So we can fake it by pretending all + * objects we are writing are the same size. This gives us a + * smooth progress meter, and at the end it matches the true + * answer. + */ + written = reuse_packfile_objects * + (((double)(total - to_write)) / total); + display_progress(progress_state, written); } close(fd); - written += reuse_packfile_objects; + written = reuse_packfile_objects; + display_progress(progress_state, written); return reuse_packfile_offset - sizeof(struct pack_header); } From 78d2214eb4d10ea1e30dd7e69a4e6d73d9f66164 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 14 Mar 2014 22:26:58 -0400 Subject: [PATCH 2/2] pack-objects: show reused packfile objects in "Counting objects" When we are sending a pack for push or fetch, we may reuse a chunk of packfile without even parsing it. The progress meter then looks like this: Reusing existing pack: 3440489, done. Counting objects: 3, done. The first line shows that we are reusing a large chunk of objects, and then we further count any objects not included in the reused portion with an actual traversal. These are all implementation details that the user does not need to care about. Instead, we can show the reused objects in the normal "counting..." progress meter (which will simply go much faster than normal), and then continue to add to it as we traverse. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index f0474db991..021ec7ad79 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1028,7 +1028,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type, exclude, name && no_try_delta(name), index_pos, found_pack, found_offset); - display_progress(progress_state, to_pack.nr_objects); + display_progress(progress_state, nr_result); return 1; } @@ -1044,7 +1044,7 @@ static int add_object_entry_from_bitmap(const unsigned char *sha1, create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset); - display_progress(progress_state, to_pack.nr_objects); + display_progress(progress_state, nr_result); return 1; } @@ -2446,12 +2446,7 @@ static int get_object_list_from_bitmap(struct rev_info *revs) &reuse_packfile_offset)) { assert(reuse_packfile_objects); nr_result += reuse_packfile_objects; - - if (progress) { - fprintf(stderr, "Reusing existing pack: %d, done.\n", - reuse_packfile_objects); - fflush(stderr); - } + display_progress(progress_state, nr_result); } traverse_bitmap_commit_list(&add_object_entry_from_bitmap);