git/reachable.c

250 строки
5.9 KiB
C
Исходник Обычный вид История

#include "cache.h"
#include "refs.h"
#include "tag.h"
#include "commit.h"
#include "blob.h"
#include "diff.h"
#include "revision.h"
#include "reachable.h"
#include "cache-tree.h"
#include "progress.h"
#include "list-objects.h"
struct connectivity_progress {
struct progress *progress;
unsigned long count;
};
static void update_progress(struct connectivity_progress *cp)
{
cp->count++;
if ((cp->count & 1023) == 0)
display_progress(cp->progress, cp->count);
}
static int add_one_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
{
struct object *object = parse_object_or_die(sha1, path);
struct rev_info *revs = (struct rev_info *)cb_data;
add_pending_object(revs, object, "");
return 0;
}
static void add_one_tree(const unsigned char *sha1, struct rev_info *revs)
{
struct tree *tree = lookup_tree(sha1);
if (tree)
add_pending_object(revs, &tree->object, "");
}
static void add_cache_tree(struct cache_tree *it, struct rev_info *revs)
{
int i;
if (it->entry_count >= 0)
add_one_tree(it->sha1, revs);
for (i = 0; i < it->subtree_nr; i++)
add_cache_tree(it->down[i]->cache_tree, revs);
}
static void add_cache_refs(struct rev_info *revs)
{
int i;
read_cache();
for (i = 0; i < active_nr; i++) {
struct blob *blob;
Make git-prune submodule aware (and fix a SEGFAULT in the process) I ran git-prune on a repository and got this: $ git-prune error: Object 228f8065b930120e35fc0c154c237487ab02d64a is a blob, not a commit Segmentation fault (core dumped) This repository was a strange one in that it was being used to provide its own submodule. That is, the repository was cloned into a subdirectory, an independent branch checked out in that subdirectory, and then it was marked as a submodule. git-prune then failed in the above manner. The problem was that git-prune was not submodule aware in two areas. Linus said: > So what happens is that something traverses a tree object, looks at each > entry, sees that it's not a tree, and tries to look it up as a blob. But > subprojects are commits, not blobs, and then when you look at the object > more closely, you get the above kind of object type confusion. and included a patch to add an S_ISGITLINK() test to reachable.c's process_tree() function. That fixed the first git-prune error, and stopped it from trying to process the gitlink entries in trees as if they were pointers to other trees (and of course failing, because gitlinks _aren't_ trees). That part of this patch is his. The second area is add_cache_refs(). This is called before starting the reachability analysis, and was calling lookup_blob() on every object hash found in the index. However, it is no longer true that every hash in the index is a pointer to a blob, some of them are gitlinks, and are not backed by any object at all, they are commits in another repository. Normally this bug was not causing any problems, but in the case of the self-referencing repository described above, it meant that the gitlink hash was being marked as being of type OBJ_BLOB by add_cache_refs() call to lookup_blob(). Then later, because that hash was also pointed to by a ref, add_one_ref() would treat it as a commit; lookup_commit() would return a NULL because that object was already noted as being an OBJ_BLOB, not an OBJ_COMMIT; and parse_commit_buffer() would SEGFAULT on that NULL pointer. The fix made by this patch is to not blindly call lookup_blob() in reachable.c's add_cache_refs(), and instead skip any index entries that are S_ISGITLINK(). Signed-off-by: Andy Parkins <andyparkins@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-07-02 16:56:58 +04:00
/*
* The index can contain blobs and GITLINKs, GITLINKs are hashes
* that don't actually point to objects in the repository, it's
* almost guaranteed that they are NOT blobs, so we don't call
* lookup_blob() on them, to avoid populating the hash table
* with invalid information
*/
if (S_ISGITLINK(active_cache[i]->ce_mode))
Make git-prune submodule aware (and fix a SEGFAULT in the process) I ran git-prune on a repository and got this: $ git-prune error: Object 228f8065b930120e35fc0c154c237487ab02d64a is a blob, not a commit Segmentation fault (core dumped) This repository was a strange one in that it was being used to provide its own submodule. That is, the repository was cloned into a subdirectory, an independent branch checked out in that subdirectory, and then it was marked as a submodule. git-prune then failed in the above manner. The problem was that git-prune was not submodule aware in two areas. Linus said: > So what happens is that something traverses a tree object, looks at each > entry, sees that it's not a tree, and tries to look it up as a blob. But > subprojects are commits, not blobs, and then when you look at the object > more closely, you get the above kind of object type confusion. and included a patch to add an S_ISGITLINK() test to reachable.c's process_tree() function. That fixed the first git-prune error, and stopped it from trying to process the gitlink entries in trees as if they were pointers to other trees (and of course failing, because gitlinks _aren't_ trees). That part of this patch is his. The second area is add_cache_refs(). This is called before starting the reachability analysis, and was calling lookup_blob() on every object hash found in the index. However, it is no longer true that every hash in the index is a pointer to a blob, some of them are gitlinks, and are not backed by any object at all, they are commits in another repository. Normally this bug was not causing any problems, but in the case of the self-referencing repository described above, it meant that the gitlink hash was being marked as being of type OBJ_BLOB by add_cache_refs() call to lookup_blob(). Then later, because that hash was also pointed to by a ref, add_one_ref() would treat it as a commit; lookup_commit() would return a NULL because that object was already noted as being an OBJ_BLOB, not an OBJ_COMMIT; and parse_commit_buffer() would SEGFAULT on that NULL pointer. The fix made by this patch is to not blindly call lookup_blob() in reachable.c's add_cache_refs(), and instead skip any index entries that are S_ISGITLINK(). Signed-off-by: Andy Parkins <andyparkins@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-07-02 16:56:58 +04:00
continue;
blob = lookup_blob(active_cache[i]->sha1);
if (blob)
blob->object.flags |= SEEN;
/*
* We could add the blobs to the pending list, but quite
* frankly, we don't care. Once we've looked them up, and
* added them as objects, we've really done everything
* there is to do for a blob
*/
}
if (active_cache_tree)
add_cache_tree(active_cache_tree, revs);
}
/*
* The traversal will have already marked us as SEEN, so we
* only need to handle any progress reporting here.
*/
static void mark_object(struct object *obj, const struct name_path *path,
const char *name, void *data)
{
update_progress(data);
}
static void mark_commit(struct commit *c, void *data)
{
mark_object(&c->object, NULL, NULL, data);
}
prune: keep objects reachable from recent objects Our current strategy with prune is that an object falls into one of three categories: 1. Reachable (from ref tips, reflogs, index, etc). 2. Not reachable, but recent (based on the --expire time). 3. Not reachable and not recent. We keep objects from (1) and (2), but prune objects in (3). The point of (2) is that these objects may be part of an in-progress operation that has not yet updated any refs. However, it is not always the case that objects for an in-progress operation will have a recent mtime. For example, the object database may have an old copy of a blob (from an abandoned operation, a branch that was deleted, etc). If we create a new tree that points to it, a simultaneous prune will leave our tree, but delete the blob. Referencing that tree with a commit will then work (we check that the tree is in the object database, but not that all of its referred objects are), as will mentioning the commit in a ref. But the resulting repo is corrupt; we are missing the blob reachable from a ref. One way to solve this is to be more thorough when referencing a sha1: make sure that not only do we have that sha1, but that we have objects it refers to, and so forth recursively. The problem is that this is very expensive. Creating a parent link would require traversing the entire object graph! Instead, this patch pushes the extra work onto prune, which runs less frequently (and has to look at the whole object graph anyway). It creates a new category of objects: objects which are not recent, but which are reachable from a recent object. We do not prune these objects, just like the reachable and recent ones. This lets us avoid the recursive check above, because if we have an object, even if it is unreachable, we should have its referent. We can make a simple inductive argument that with this patch, this property holds (that there are no objects with missing referents in the repository): 0. When we have no objects, we have nothing to refer or be referred to, so the property holds. 1. If we add objects to the repository, their direct referents must generally exist (e.g., if you create a tree, the blobs it references must exist; if you create a commit to point at the tree, the tree must exist). This is already the case before this patch. And it is not 100% foolproof (you can make bogus objects using `git hash-object`, for example), but it should be the case for normal usage. Therefore for any sequence of object additions, the property will continue to hold. 2. If we remove objects from the repository, then we will not remove a child object (like a blob) if an object that refers to it is being kept. That is the part implemented by this patch. Note, however, that our reachability check and the actual pruning are not atomic. So it _is_ still possible to violate the property (e.g., an object becomes referenced just as we are deleting it). This patch is shooting for eliminating problems where the mtimes of dependent objects differ by hours or days, and one is dropped without the other. It does nothing to help with short races. Naively, the simplest way to implement this would be to add all recent objects as tips to the reachability traversal. However, this does not perform well. In a recently-packed repository, all reachable objects will also be recent, and therefore we have to look at each object twice. This patch instead performs the reachability traversal, then follows up with a second traversal for recent objects, skipping any that have already been marked. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-16 02:41:35 +04:00
struct recent_data {
struct rev_info *revs;
unsigned long timestamp;
};
static void add_recent_object(const unsigned char *sha1,
unsigned long mtime,
struct recent_data *data)
{
struct object *obj;
enum object_type type;
if (mtime <= data->timestamp)
return;
/*
* We do not want to call parse_object here, because
* inflating blobs and trees could be very expensive.
* However, we do need to know the correct type for
* later processing, and the revision machinery expects
* commits and tags to have been parsed.
*/
type = sha1_object_info(sha1, NULL);
if (type < 0)
die("unable to get object info for %s", sha1_to_hex(sha1));
switch (type) {
case OBJ_TAG:
case OBJ_COMMIT:
obj = parse_object_or_die(sha1, NULL);
break;
case OBJ_TREE:
obj = (struct object *)lookup_tree(sha1);
break;
case OBJ_BLOB:
obj = (struct object *)lookup_blob(sha1);
break;
default:
die("unknown object type for %s: %s",
sha1_to_hex(sha1), typename(type));
}
if (!obj)
die("unable to lookup %s", sha1_to_hex(sha1));
add_pending_object(data->revs, obj, "");
}
static int add_recent_loose(const unsigned char *sha1,
const char *path, void *data)
{
struct stat st;
struct object *obj = lookup_object(sha1);
if (obj && obj->flags & SEEN)
return 0;
if (stat(path, &st) < 0) {
/*
* It's OK if an object went away during our iteration; this
* could be due to a simultaneous repack. But anything else
* we should abort, since we might then fail to mark objects
* which should not be pruned.
*/
if (errno == ENOENT)
return 0;
return error("unable to stat %s: %s",
sha1_to_hex(sha1), strerror(errno));
}
add_recent_object(sha1, st.st_mtime, data);
return 0;
}
static int add_recent_packed(const unsigned char *sha1,
struct packed_git *p, uint32_t pos,
void *data)
{
struct object *obj = lookup_object(sha1);
if (obj && obj->flags & SEEN)
return 0;
add_recent_object(sha1, p->mtime, data);
return 0;
}
int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
unsigned long timestamp)
prune: keep objects reachable from recent objects Our current strategy with prune is that an object falls into one of three categories: 1. Reachable (from ref tips, reflogs, index, etc). 2. Not reachable, but recent (based on the --expire time). 3. Not reachable and not recent. We keep objects from (1) and (2), but prune objects in (3). The point of (2) is that these objects may be part of an in-progress operation that has not yet updated any refs. However, it is not always the case that objects for an in-progress operation will have a recent mtime. For example, the object database may have an old copy of a blob (from an abandoned operation, a branch that was deleted, etc). If we create a new tree that points to it, a simultaneous prune will leave our tree, but delete the blob. Referencing that tree with a commit will then work (we check that the tree is in the object database, but not that all of its referred objects are), as will mentioning the commit in a ref. But the resulting repo is corrupt; we are missing the blob reachable from a ref. One way to solve this is to be more thorough when referencing a sha1: make sure that not only do we have that sha1, but that we have objects it refers to, and so forth recursively. The problem is that this is very expensive. Creating a parent link would require traversing the entire object graph! Instead, this patch pushes the extra work onto prune, which runs less frequently (and has to look at the whole object graph anyway). It creates a new category of objects: objects which are not recent, but which are reachable from a recent object. We do not prune these objects, just like the reachable and recent ones. This lets us avoid the recursive check above, because if we have an object, even if it is unreachable, we should have its referent. We can make a simple inductive argument that with this patch, this property holds (that there are no objects with missing referents in the repository): 0. When we have no objects, we have nothing to refer or be referred to, so the property holds. 1. If we add objects to the repository, their direct referents must generally exist (e.g., if you create a tree, the blobs it references must exist; if you create a commit to point at the tree, the tree must exist). This is already the case before this patch. And it is not 100% foolproof (you can make bogus objects using `git hash-object`, for example), but it should be the case for normal usage. Therefore for any sequence of object additions, the property will continue to hold. 2. If we remove objects from the repository, then we will not remove a child object (like a blob) if an object that refers to it is being kept. That is the part implemented by this patch. Note, however, that our reachability check and the actual pruning are not atomic. So it _is_ still possible to violate the property (e.g., an object becomes referenced just as we are deleting it). This patch is shooting for eliminating problems where the mtimes of dependent objects differ by hours or days, and one is dropped without the other. It does nothing to help with short races. Naively, the simplest way to implement this would be to add all recent objects as tips to the reachability traversal. However, this does not perform well. In a recently-packed repository, all reachable objects will also be recent, and therefore we have to look at each object twice. This patch instead performs the reachability traversal, then follows up with a second traversal for recent objects, skipping any that have already been marked. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-16 02:41:35 +04:00
{
struct recent_data data;
int r;
data.revs = revs;
data.timestamp = timestamp;
r = for_each_loose_object(add_recent_loose, &data);
if (r)
return r;
return for_each_packed_object(add_recent_packed, &data);
}
void mark_reachable_objects(struct rev_info *revs, int mark_reflog,
prune: keep objects reachable from recent objects Our current strategy with prune is that an object falls into one of three categories: 1. Reachable (from ref tips, reflogs, index, etc). 2. Not reachable, but recent (based on the --expire time). 3. Not reachable and not recent. We keep objects from (1) and (2), but prune objects in (3). The point of (2) is that these objects may be part of an in-progress operation that has not yet updated any refs. However, it is not always the case that objects for an in-progress operation will have a recent mtime. For example, the object database may have an old copy of a blob (from an abandoned operation, a branch that was deleted, etc). If we create a new tree that points to it, a simultaneous prune will leave our tree, but delete the blob. Referencing that tree with a commit will then work (we check that the tree is in the object database, but not that all of its referred objects are), as will mentioning the commit in a ref. But the resulting repo is corrupt; we are missing the blob reachable from a ref. One way to solve this is to be more thorough when referencing a sha1: make sure that not only do we have that sha1, but that we have objects it refers to, and so forth recursively. The problem is that this is very expensive. Creating a parent link would require traversing the entire object graph! Instead, this patch pushes the extra work onto prune, which runs less frequently (and has to look at the whole object graph anyway). It creates a new category of objects: objects which are not recent, but which are reachable from a recent object. We do not prune these objects, just like the reachable and recent ones. This lets us avoid the recursive check above, because if we have an object, even if it is unreachable, we should have its referent. We can make a simple inductive argument that with this patch, this property holds (that there are no objects with missing referents in the repository): 0. When we have no objects, we have nothing to refer or be referred to, so the property holds. 1. If we add objects to the repository, their direct referents must generally exist (e.g., if you create a tree, the blobs it references must exist; if you create a commit to point at the tree, the tree must exist). This is already the case before this patch. And it is not 100% foolproof (you can make bogus objects using `git hash-object`, for example), but it should be the case for normal usage. Therefore for any sequence of object additions, the property will continue to hold. 2. If we remove objects from the repository, then we will not remove a child object (like a blob) if an object that refers to it is being kept. That is the part implemented by this patch. Note, however, that our reachability check and the actual pruning are not atomic. So it _is_ still possible to violate the property (e.g., an object becomes referenced just as we are deleting it). This patch is shooting for eliminating problems where the mtimes of dependent objects differ by hours or days, and one is dropped without the other. It does nothing to help with short races. Naively, the simplest way to implement this would be to add all recent objects as tips to the reachability traversal. However, this does not perform well. In a recently-packed repository, all reachable objects will also be recent, and therefore we have to look at each object twice. This patch instead performs the reachability traversal, then follows up with a second traversal for recent objects, skipping any that have already been marked. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-16 02:41:35 +04:00
unsigned long mark_recent,
struct progress *progress)
{
struct connectivity_progress cp;
/*
* Set up revision parsing, and mark us as being interested
* in all object types, not just commits.
*/
revs->tag_objects = 1;
revs->blob_objects = 1;
revs->tree_objects = 1;
/* Add all refs from the index file */
add_cache_refs(revs);
/* Add all external refs */
for_each_ref(add_one_ref, revs);
/* detached HEAD is not included in the list above */
head_ref(add_one_ref, revs);
/* Add all reflog info */
if (mark_reflog)
add_reflogs_to_pending(revs, 0);
cp.progress = progress;
cp.count = 0;
/*
* Set up the revision walk - this will move all commits
* from the pending list to the commit walking list.
*/
if (prepare_revision_walk(revs))
die("revision walk setup failed");
traverse_commit_list(revs, mark_commit, mark_object, &cp);
prune: keep objects reachable from recent objects Our current strategy with prune is that an object falls into one of three categories: 1. Reachable (from ref tips, reflogs, index, etc). 2. Not reachable, but recent (based on the --expire time). 3. Not reachable and not recent. We keep objects from (1) and (2), but prune objects in (3). The point of (2) is that these objects may be part of an in-progress operation that has not yet updated any refs. However, it is not always the case that objects for an in-progress operation will have a recent mtime. For example, the object database may have an old copy of a blob (from an abandoned operation, a branch that was deleted, etc). If we create a new tree that points to it, a simultaneous prune will leave our tree, but delete the blob. Referencing that tree with a commit will then work (we check that the tree is in the object database, but not that all of its referred objects are), as will mentioning the commit in a ref. But the resulting repo is corrupt; we are missing the blob reachable from a ref. One way to solve this is to be more thorough when referencing a sha1: make sure that not only do we have that sha1, but that we have objects it refers to, and so forth recursively. The problem is that this is very expensive. Creating a parent link would require traversing the entire object graph! Instead, this patch pushes the extra work onto prune, which runs less frequently (and has to look at the whole object graph anyway). It creates a new category of objects: objects which are not recent, but which are reachable from a recent object. We do not prune these objects, just like the reachable and recent ones. This lets us avoid the recursive check above, because if we have an object, even if it is unreachable, we should have its referent. We can make a simple inductive argument that with this patch, this property holds (that there are no objects with missing referents in the repository): 0. When we have no objects, we have nothing to refer or be referred to, so the property holds. 1. If we add objects to the repository, their direct referents must generally exist (e.g., if you create a tree, the blobs it references must exist; if you create a commit to point at the tree, the tree must exist). This is already the case before this patch. And it is not 100% foolproof (you can make bogus objects using `git hash-object`, for example), but it should be the case for normal usage. Therefore for any sequence of object additions, the property will continue to hold. 2. If we remove objects from the repository, then we will not remove a child object (like a blob) if an object that refers to it is being kept. That is the part implemented by this patch. Note, however, that our reachability check and the actual pruning are not atomic. So it _is_ still possible to violate the property (e.g., an object becomes referenced just as we are deleting it). This patch is shooting for eliminating problems where the mtimes of dependent objects differ by hours or days, and one is dropped without the other. It does nothing to help with short races. Naively, the simplest way to implement this would be to add all recent objects as tips to the reachability traversal. However, this does not perform well. In a recently-packed repository, all reachable objects will also be recent, and therefore we have to look at each object twice. This patch instead performs the reachability traversal, then follows up with a second traversal for recent objects, skipping any that have already been marked. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2014-10-16 02:41:35 +04:00
if (mark_recent) {
revs->ignore_missing_links = 1;
if (add_unseen_recent_objects_to_traversal(revs, mark_recent))
die("unable to mark recent objects");
if (prepare_revision_walk(revs))
die("revision walk setup failed");
traverse_commit_list(revs, mark_commit, mark_object, &cp);
}
display_progress(cp.progress, cp.count);
}