diff --git a/builtin-remote.c b/builtin-remote.c
index e5cfc882b5..c49f00f58b 100644
--- a/builtin-remote.c
+++ b/builtin-remote.c
@@ -267,28 +267,66 @@ static int get_ref_states(const struct ref *ref, struct ref_states *states)
 	return 0;
 }
 
+struct known_remote {
+	struct known_remote *next;
+	struct remote *remote;
+};
+
+struct known_remotes {
+	struct remote *to_delete;
+	struct known_remote *list;
+};
+
+static int add_known_remote(struct remote *remote, void *cb_data)
+{
+	struct known_remotes *all = cb_data;
+	struct known_remote *r;
+
+	if (!strcmp(all->to_delete->name, remote->name))
+		return 0;
+
+	r = xmalloc(sizeof(*r));
+	r->remote = remote;
+	r->next = all->list;
+	all->list = r;
+	return 0;
+}
+
 struct branches_for_remote {
-	const char *prefix;
+	struct remote *remote;
 	struct path_list *branches;
+	struct known_remotes *keep;
 };
 
 static int add_branch_for_removal(const char *refname,
 	const unsigned char *sha1, int flags, void *cb_data)
 {
 	struct branches_for_remote *branches = cb_data;
+	struct refspec refspec;
+	struct path_list_item *item;
+	struct known_remote *kr;
 
-	if (!prefixcmp(refname, branches->prefix)) {
-		struct path_list_item *item;
+	memset(&refspec, 0, sizeof(refspec));
+	refspec.dst = (char *)refname;
+	if (remote_find_tracking(branches->remote, &refspec))
+		return 0;
 
-		/* make sure that symrefs are deleted */
-		if (flags & REF_ISSYMREF)
-			return unlink(git_path(refname));
-
-		item = path_list_append(refname, branches->branches);
-		item->util = xmalloc(20);
-		hashcpy(item->util, sha1);
+	/* don't delete a branch if another remote also uses it */
+	for (kr = branches->keep->list; kr; kr = kr->next) {
+		memset(&refspec, 0, sizeof(refspec));
+		refspec.dst = (char *)refname;
+		if (!remote_find_tracking(kr->remote, &refspec))
+			return 0;
 	}
 
+	/* make sure that symrefs are deleted */
+	if (flags & REF_ISSYMREF)
+		return unlink(git_path(refname));
+
+	item = path_list_append(refname, branches->branches);
+	item->util = xmalloc(20);
+	hashcpy(item->util, sha1);
+
 	return 0;
 }
 
@@ -313,8 +351,9 @@ static int rm(int argc, const char **argv)
 	};
 	struct remote *remote;
 	struct strbuf buf;
+	struct known_remotes known_remotes = { NULL, NULL };
 	struct path_list branches = { NULL, 0, 0, 1 };
-	struct branches_for_remote cb_data = { NULL, &branches };
+	struct branches_for_remote cb_data = { NULL, &branches, &known_remotes };
 	int i;
 
 	if (argc != 2)
@@ -324,6 +363,9 @@ static int rm(int argc, const char **argv)
 	if (!remote)
 		die("No such remote: %s", argv[1]);
 
+	known_remotes.to_delete = remote;
+	for_each_remote(add_known_remote, &known_remotes);
+
 	strbuf_init(&buf, 0);
 	strbuf_addf(&buf, "remote.%s", remote->name);
 	if (git_config_rename_section(buf.buf, NULL) < 1)
@@ -352,9 +394,7 @@ static int rm(int argc, const char **argv)
 	 * the branches one by one, since for_each_ref() relies on cached
 	 * refs, which are invalidated when deleting a branch.
 	 */
-	strbuf_reset(&buf);
-	strbuf_addf(&buf, "refs/remotes/%s/", remote->name);
-	cb_data.prefix = buf.buf;
+	cb_data.remote = remote;
 	i = for_each_ref(add_branch_for_removal, &cb_data);
 	strbuf_release(&buf);