Merge branch 'sp/fetch-optim'

* sp/fetch-optim:
  Teach git-fetch to exploit server side automatic tag following
  Teach fetch-pack/upload-pack about --include-tag
  git-pack-objects: Automatically pack annotated tags if object was packed
  Teach git-fetch to grab a tag at the same time as a commit
  Make git-fetch follow tags we already have objects for sooner
  Teach upload-pack to log the received need lines to an fd
  Free the path_lists used to find non-local tags in git-fetch
  Allow builtin-fetch's find_non_local_tags to append onto a list
  Ensure tail pointer gets setup correctly when we fetch HEAD only
  Remove unnecessary delaying of free_refs(ref_map) in builtin-fetch
  Remove unused variable in builtin-fetch find_non_local_tags

Documentation/git-fetch-pack.txt

@@ -8,7 +8,7 @@ git-fetch-pack - Receive missing objects from another repository
 
 SYNOPSIS
 --------
-'git-fetch-pack' [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]
+'git-fetch-pack' [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]
 
 DESCRIPTION
 -----------
@@ -45,6 +45,12 @@ OPTIONS
 	Spend extra cycles to minimize the number of objects to be sent.
 	Use it on slower connection.
 
+\--include-tag::
+	If the remote side supports it, annotated tags objects will
+	be downloaded on the same connection as the other objects if
+	the object the tag references is downloaded.  The caller must
+	otherwise determine the tags this option made available.
+
 \--upload-pack=<git-upload-pack>::
 	Use this to specify the path to 'git-upload-pack' on the
 	remote side, if is not found on your $PATH.

Documentation/git-pack-objects.txt

@@ -73,6 +73,11 @@ base-name::
 	as if all refs under `$GIT_DIR/refs` are specified to be
 	included.
 
+--include-tag::
+	Include unasked-for annotated tags if the object they
+	reference was included in the resulting packfile.  This
+	can be useful to send new tags to native git clients.
+
 --window=[N], --depth=[N]::
 	These two options affect how the objects contained in
 	the pack are stored using delta compression.  The

builtin-fetch-pack.c

@@ -18,7 +18,7 @@ static struct fetch_pack_args args = {
 };
 
 static const char fetch_pack_usage[] =
-"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
+"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
 
 #define COMPLETE	(1U << 0)
 #define COMMON		(1U << 1)
@@ -176,13 +176,14 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 		}
 
 		if (!fetching)
-			packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
+			packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
 				     sha1_to_hex(remote),
 				     (multi_ack ? " multi_ack" : ""),
 				     (use_sideband == 2 ? " side-band-64k" : ""),
 				     (use_sideband == 1 ? " side-band" : ""),
 				     (args.use_thin_pack ? " thin-pack" : ""),
 				     (args.no_progress ? " no-progress" : ""),
+				     (args.include_tag ? " include-tag" : ""),
 				     " ofs-delta");
 		else
 			packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
@@ -683,6 +684,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 				args.use_thin_pack = 1;
 				continue;
 			}
+			if (!strcmp("--include-tag", arg)) {
+				args.include_tag = 1;
+				continue;
+			}
 			if (!strcmp("--all", arg)) {
 				args.fetch_all = 1;
 				continue;

builtin-fetch.c

@@ -101,6 +101,10 @@ static void add_merge_config(struct ref **head,
 	}
 }
 
+static void find_non_local_tags(struct transport *transport,
+			struct ref **head,
+			struct ref ***tail);
+
 static struct ref *get_ref_map(struct transport *transport,
 			       struct refspec *refs, int ref_count, int tags,
 			       int *autotags)
@@ -157,8 +161,11 @@ static struct ref *get_ref_map(struct transport *transport,
 			if (!ref_map)
 				die("Couldn't find remote ref HEAD");
 			ref_map->merge = 1;
+			tail = &ref_map->next;
 		}
 	}
+	if (tags == TAGS_DEFAULT && *autotags)
+		find_non_local_tags(transport, &ref_map, &tail);
 	ref_remove_duplicates(ref_map);
 
 	return ref_map;
@@ -452,18 +459,28 @@ static int add_existing(const char *refname, const unsigned char *sha1,
 	return 0;
 }
 
-static struct ref *find_non_local_tags(struct transport *transport,
-				       struct ref *fetch_map)
+static int will_fetch(struct ref **head, const unsigned char *sha1)
 {
-	static struct path_list existing_refs = { NULL, 0, 0, 0 };
+	struct ref *rm = *head;
+	while (rm) {
+		if (!hashcmp(rm->old_sha1, sha1))
+			return 1;
+		rm = rm->next;
+	}
+	return 0;
+}
+
+static void find_non_local_tags(struct transport *transport,
+			struct ref **head,
+			struct ref ***tail)
+{
+	struct path_list existing_refs = { NULL, 0, 0, 0 };
 	struct path_list new_refs = { NULL, 0, 0, 1 };
 	char *ref_name;
 	int ref_name_len;
 	const unsigned char *ref_sha1;
 	const struct ref *tag_ref;
 	struct ref *rm = NULL;
-	struct ref *ref_map = NULL;
-	struct ref **tail = &ref_map;
 	const struct ref *ref;
 
 	for_each_ref(add_existing, &existing_refs);
@@ -489,7 +506,8 @@ static struct ref *find_non_local_tags(struct transport *transport,
 
 		if (!path_list_has_path(&existing_refs, ref_name) &&
 		    !path_list_has_path(&new_refs, ref_name) &&
-		    has_sha1_file(ref->old_sha1)) {
+		    (has_sha1_file(ref->old_sha1) ||
+		     will_fetch(head, ref->old_sha1))) {
 			path_list_insert(ref_name, &new_refs);
 
 			rm = alloc_ref(strlen(ref_name) + 1);
@@ -498,19 +516,19 @@ static struct ref *find_non_local_tags(struct transport *transport,
 			strcpy(rm->peer_ref->name, ref_name);
 			hashcpy(rm->old_sha1, ref_sha1);
 
-			*tail = rm;
-			tail = &rm->next;
+			**tail = rm;
+			*tail = &rm->next;
 		}
 		free(ref_name);
 	}
-
-	return ref_map;
+	path_list_clear(&existing_refs, 0);
+	path_list_clear(&new_refs, 0);
 }
 
 static int do_fetch(struct transport *transport,
 		    struct refspec *refs, int ref_count)
 {
-	struct ref *ref_map, *fetch_map;
+	struct ref *ref_map;
 	struct ref *rm;
 	int autotags = (transport->remote->fetch_tags == 1);
 	if (transport->remote->fetch_tags == 2 && tags != TAGS_UNSET)
@@ -537,26 +555,28 @@ static int do_fetch(struct transport *transport,
 			read_ref(rm->peer_ref->name, rm->peer_ref->old_sha1);
 	}
 
+	if (tags == TAGS_DEFAULT && autotags)
+		transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1");
 	if (fetch_refs(transport, ref_map)) {
 		free_refs(ref_map);
 		return 1;
 	}
-
-	fetch_map = ref_map;
+	free_refs(ref_map);
 
 	/* if neither --no-tags nor --tags was specified, do automated tag
 	 * following ... */
 	if (tags == TAGS_DEFAULT && autotags) {
-		ref_map = find_non_local_tags(transport, fetch_map);
+		struct ref **tail = &ref_map;
+		ref_map = NULL;
+		find_non_local_tags(transport, &ref_map, &tail);
 		if (ref_map) {
+			transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
 			transport_set_option(transport, TRANS_OPT_DEPTH, "0");
 			fetch_refs(transport, ref_map);
 		}
 		free_refs(ref_map);
 	}
 
-	free_refs(fetch_map);
-
 	transport_disconnect(transport);
 
 	return 0;

builtin-pack-objects.c

@@ -15,6 +15,7 @@
 #include "revision.h"
 #include "list-objects.h"
 #include "progress.h"
+#include "refs.h"
 
 #ifdef THREADED_DELTA_SEARCH
 #include "thread-utils.h"
@@ -27,7 +28,8 @@ git-pack-objects [{ -q | --progress | --all-progress }] \n\
 	[--window=N] [--window-memory=N] [--depth=N] \n\
 	[--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
 	[--threads=N] [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
-	[--stdout | base-name] [--keep-unreachable] [<ref-list | <object-list]";
+	[--stdout | base-name] [--include-tag] [--keep-unreachable] \n\
+	[<ref-list | <object-list]";
 
 struct object_entry {
 	struct pack_idx_entry idx;
@@ -63,7 +65,7 @@ static struct pack_idx_entry **written_list;
 static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
 
 static int non_empty;
-static int no_reuse_delta, no_reuse_object, keep_unreachable;
+static int no_reuse_delta, no_reuse_object, keep_unreachable, include_tag;
 static int local;
 static int incremental;
 static int allow_ofs_delta;
@@ -1630,6 +1632,18 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size,
 #define ll_find_deltas(l, s, w, d, p)	find_deltas(l, &s, w, d, p)
 #endif
 
+static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, void *cb_data)
+{
+	unsigned char peeled[20];
+
+	if (!prefixcmp(path, "refs/tags/") && /* is a tag? */
+	    !peel_ref(path, peeled)        && /* peelable? */
+	    !is_null_sha1(peeled)          && /* annotated tag? */
+	    locate_object_entry(peeled))      /* object packed? */
+		add_object_entry(sha1, OBJ_TAG, NULL, 0);
+	return 0;
+}
+
 static void prepare_pack(int window, int depth)
 {
 	struct object_entry **delta_list;
@@ -2033,6 +2047,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 			keep_unreachable = 1;
 			continue;
 		}
+		if (!strcmp("--include-tag", arg)) {
+			include_tag = 1;
+			continue;
+		}
 		if (!strcmp("--unpacked", arg) ||
 		    !prefixcmp(arg, "--unpacked=") ||
 		    !strcmp("--reflog", arg) ||
@@ -2109,6 +2127,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 		rp_av[rp_ac] = NULL;
 		get_object_list(rp_ac, rp_av);
 	}
+	if (include_tag && nr_result)
+		for_each_ref(add_ref_tag, NULL);
 	stop_progress(&progress_state);
 
 	if (non_empty && !nr_result)

fetch-pack.h

@@ -12,7 +12,8 @@ struct fetch_pack_args
 		use_thin_pack:1,
 		fetch_all:1,
 		verbose:1,
-		no_progress:1;
+		no_progress:1,
+		include_tag:1;
 };
 
 struct ref *fetch_pack(struct fetch_pack_args *args,

t/t5305-include-tag.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+test_description='git-pack-object --include-tag'
+. ./test-lib.sh
+
+TRASH=`pwd`
+
+test_expect_success setup '
+	echo c >d &&
+	git update-index --add d &&
+	tree=`git write-tree` &&
+	commit=`git commit-tree $tree </dev/null` &&
+	echo "object $commit" >sig &&
+	echo "type commit" >>sig &&
+	echo "tag mytag" >>sig &&
+	echo "tagger $(git var GIT_COMMITTER_IDENT)" >>sig &&
+	echo >>sig &&
+	echo "our test tag" >>sig &&
+	tag=`git mktag <sig` &&
+	rm d sig &&
+	git update-ref refs/tags/mytag $tag && {
+		echo $tree &&
+		echo $commit &&
+		git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)	.*/\\1/"
+	} >obj-list
+'
+
+rm -rf clone.git
+test_expect_success 'pack without --include-tag' '
+	packname_1=$(git pack-objects \
+		--window=0 \
+		test-1 <obj-list)
+'
+
+test_expect_success 'unpack objects' '
+	(
+		GIT_DIR=clone.git &&
+		export GIT_DIR &&
+		git init &&
+		git unpack-objects -n <test-1-${packname_1}.pack &&
+		git unpack-objects <test-1-${packname_1}.pack
+	)
+'
+
+test_expect_success 'check unpacked result (have commit, no tag)' '
+	git rev-list --objects $commit >list.expect &&
+	(
+		GIT_DIR=clone.git &&
+		export GIT_DIR &&
+		test_must_fail git cat-file -e $tag &&
+		git rev-list --objects $commit
+	) >list.actual &&
+	git diff list.expect list.actual
+'
+
+rm -rf clone.git
+test_expect_success 'pack with --include-tag' '
+	packname_1=$(git pack-objects \
+		--window=0 \
+		--include-tag \
+		test-2 <obj-list)
+'
+
+test_expect_success 'unpack objects' '
+	(
+		GIT_DIR=clone.git &&
+		export GIT_DIR &&
+		git init &&
+		git unpack-objects -n <test-2-${packname_1}.pack &&
+		git unpack-objects <test-2-${packname_1}.pack
+	)
+'
+
+test_expect_success 'check unpacked result (have commit, have tag)' '
+	git rev-list --objects mytag >list.expect &&
+	(
+		GIT_DIR=clone.git &&
+		export GIT_DIR &&
+		git rev-list --objects $tag
+	) >list.actual &&
+	git diff list.expect list.actual
+'
+
+test_done

t/t5503-tagfollow.sh

@@ -0,0 +1,150 @@
+#!/bin/sh
+
+test_description='test automatic tag following'
+
+. ./test-lib.sh
+
+# End state of the repository:
+#
+#         T - tag1          S - tag2
+#        /                 /
+#   L - A ------ O ------ B
+#    \   \                 \
+#     \   C - origin/cat    \
+#      origin/master         master
+
+test_expect_success setup '
+	test_tick &&
+	echo ichi >file &&
+	git add file &&
+	git commit -m L &&
+	L=$(git rev-parse --verify HEAD) &&
+
+	(
+		mkdir cloned &&
+		cd cloned &&
+		git init-db &&
+		git remote add -f origin ..
+	) &&
+
+	test_tick &&
+	echo A >file &&
+	git add file &&
+	git commit -m A &&
+	A=$(git rev-parse --verify HEAD)
+'
+
+U=UPLOAD_LOG
+
+cat - <<EOF >expect
+#S
+want $A
+#E
+EOF
+test_expect_success 'fetch A (new commit : 1 connection)' '
+	rm -f $U
+	(
+		cd cloned &&
+		GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U &&
+		test $A = $(git rev-parse --verify origin/master)
+	) &&
+	test -s $U &&
+	cut -d" " -f1,2 $U >actual &&
+	git diff expect actual
+'
+
+test_expect_success "create tag T on A, create C on branch cat" '
+	git tag -a -m tag1 tag1 $A &&
+	T=$(git rev-parse --verify tag1) &&
+
+	git checkout -b cat &&
+	echo C >file &&
+	git add file &&
+	git commit -m C &&
+	C=$(git rev-parse --verify HEAD) &&
+	git checkout master
+'
+
+cat - <<EOF >expect
+#S
+want $C
+want $T
+#E
+EOF
+test_expect_success 'fetch C, T (new branch, tag : 1 connection)' '
+	rm -f $U
+	(
+		cd cloned &&
+		GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U &&
+		test $C = $(git rev-parse --verify origin/cat) &&
+		test $T = $(git rev-parse --verify tag1) &&
+		test $A = $(git rev-parse --verify tag1^0)
+	) &&
+	test -s $U &&
+	cut -d" " -f1,2 $U >actual &&
+	git diff expect actual
+'
+
+test_expect_success "create commits O, B, tag S on B" '
+	test_tick &&
+	echo O >file &&
+	git add file &&
+	git commit -m O &&
+
+	test_tick &&
+	echo B >file &&
+	git add file &&
+	git commit -m B &&
+	B=$(git rev-parse --verify HEAD) &&
+
+	git tag -a -m tag2 tag2 $B &&
+	S=$(git rev-parse --verify tag2)
+'
+
+cat - <<EOF >expect
+#S
+want $B
+want $S
+#E
+EOF
+test_expect_success 'fetch B, S (commit and tag : 1 connection)' '
+	rm -f $U
+	(
+		cd cloned &&
+		GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U &&
+		test $B = $(git rev-parse --verify origin/master) &&
+		test $B = $(git rev-parse --verify tag2^0) &&
+		test $S = $(git rev-parse --verify tag2)
+	) &&
+	test -s $U &&
+	cut -d" " -f1,2 $U >actual &&
+	git diff expect actual
+'
+
+cat - <<EOF >expect
+#S
+want $B
+want $S
+#E
+EOF
+test_expect_success 'new clone fetch master and tags' '
+	git branch -D cat
+	rm -f $U
+	(
+		mkdir clone2 &&
+		cd clone2 &&
+		git init &&
+		git remote add origin .. &&
+		GIT_DEBUG_SEND_PACK=3 git fetch 3>../$U &&
+		test $B = $(git rev-parse --verify origin/master) &&
+		test $S = $(git rev-parse --verify tag2) &&
+		test $B = $(git rev-parse --verify tag2^0) &&
+		test $T = $(git rev-parse --verify tag1) &&
+		test $A = $(git rev-parse --verify tag1^0)
+	) &&
+	test -s $U &&
+	cut -d" " -f1,2 $U >actual &&
+	git diff expect actual
+'
+
+test_done

transport.c

@@ -560,6 +560,7 @@ static int close_bundle(struct transport *transport)
 struct git_transport_data {
 	unsigned thin : 1;
 	unsigned keep : 1;
+	unsigned followtags : 1;
 	int depth;
 	struct child_process *conn;
 	int fd[2];
@@ -580,6 +581,9 @@ static int set_git_option(struct transport *connection,
 	} else if (!strcmp(name, TRANS_OPT_THIN)) {
 		data->thin = !!value;
 		return 0;
+	} else if (!strcmp(name, TRANS_OPT_FOLLOWTAGS)) {
+		data->followtags = !!value;
+		return 0;
 	} else if (!strcmp(name, TRANS_OPT_KEEP)) {
 		data->keep = !!value;
 		return 0;
@@ -628,6 +632,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 	args.keep_pack = data->keep;
 	args.lock_pack = 1;
 	args.use_thin_pack = data->thin;
+	args.include_tag = data->followtags;
 	args.verbose = transport->verbose > 0;
 	args.depth = data->depth;
 

transport.h

@@ -53,6 +53,9 @@ struct transport *transport_get(struct remote *, const char *);
 /* Limit the depth of the fetch if not null */
 #define TRANS_OPT_DEPTH "depth"
 
+/* Aggressively fetch annotated tags if possible */
+#define TRANS_OPT_FOLLOWTAGS "followtags"
+
 /**
  * Returns 0 if the option was used, non-zero otherwise. Prints a
  * message to stderr if the option is not used.

upload-pack.c

@@ -27,7 +27,8 @@ static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=n
 static unsigned long oldest_have;
 
 static int multi_ack, nr_our_refs;
-static int use_thin_pack, use_ofs_delta, no_progress;
+static int use_thin_pack, use_ofs_delta, use_include_tag;
+static int no_progress;
 static struct object_array have_obj;
 static struct object_array want_obj;
 static unsigned int timeout;
@@ -35,6 +36,7 @@ static unsigned int timeout;
  * otherwise maximum packet size (up to 65520 bytes).
  */
 static int use_sideband;
+static int debug_fd;
 
 static void reset_timeout(void)
 {
@@ -161,6 +163,8 @@ static void create_pack_file(void)
 		argv[arg++] = "--progress";
 	if (use_ofs_delta)
 		argv[arg++] = "--delta-base-offset";
+	if (use_include_tag)
+		argv[arg++] = "--include-tag";
 	argv[arg++] = NULL;
 
 	memset(&pack_objects, 0, sizeof(pack_objects));
@@ -444,6 +448,8 @@ static void receive_needs(void)
 	static char line[1000];
 	int len, depth = 0;
 
+	if (debug_fd)
+		write_in_full(debug_fd, "#S\n", 3);
 	for (;;) {
 		struct object *o;
 		unsigned char sha1_buf[20];
@@ -451,6 +457,8 @@ static void receive_needs(void)
 		reset_timeout();
 		if (!len)
 			break;
+		if (debug_fd)
+			write_in_full(debug_fd, line, len);
 
 		if (!prefixcmp(line, "shallow ")) {
 			unsigned char sha1[20];
@@ -489,6 +497,8 @@ static void receive_needs(void)
 			use_sideband = DEFAULT_PACKET_MAX;
 		if (strstr(line+45, "no-progress"))
 			no_progress = 1;
+		if (strstr(line+45, "include-tag"))
+			use_include_tag = 1;
 
 		/* We have sent all our refs already, and the other end
 		 * should have chosen out of them; otherwise they are
@@ -506,6 +516,8 @@ static void receive_needs(void)
 			add_object_array(o, NULL, &want_obj);
 		}
 	}
+	if (debug_fd)
+		write_in_full(debug_fd, "#E\n", 3);
 	if (depth == 0 && shallows.nr == 0)
 		return;
 	if (depth > 0) {
@@ -558,7 +570,8 @@ static void receive_needs(void)
 static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 {
 	static const char *capabilities = "multi_ack thin-pack side-band"
-		" side-band-64k ofs-delta shallow no-progress";
+		" side-band-64k ofs-delta shallow no-progress"
+		" include-tag";
 	struct object *o = parse_object(sha1);
 
 	if (!o)
@@ -631,6 +644,8 @@ int main(int argc, char **argv)
 		die("'%s': unable to chdir or not a git archive", dir);
 	if (is_repository_shallow())
 		die("attempt to fetch/clone from a shallow repository");
+	if (getenv("GIT_DEBUG_SEND_PACK"))
+		debug_fd = atoi(getenv("GIT_DEBUG_SEND_PACK"));
 	upload_pack();
 	return 0;
 }