git/sha1_file.c

2838 строки
71 KiB
C
Исходник Обычный вид История

/*
* GIT - The information manager from hell
*
* Copyright (C) Linus Torvalds, 2005
*
* This handles basic git sha1 object files - packing, unpacking,
* creation etc.
*/
#include "cache.h"
#include "string-list.h"
#include "delta.h"
#include "pack.h"
#include "blob.h"
#include "commit.h"
#include "run-command.h"
#include "tag.h"
#include "tree.h"
#include "tree-walk.h"
#include "refs.h"
#include "pack-revindex.h"
sha1-lookup: more memory efficient search in sorted list of SHA-1 Currently, when looking for a packed object from the pack idx, a simple binary search is used. A conventional binary search loop looks like this: unsigned lo, hi; do { unsigned mi = (lo + hi) / 2; int cmp = "entry pointed at by mi" minus "target"; if (!cmp) return mi; "mi is the wanted one" if (cmp > 0) hi = mi; "mi is larger than target" else lo = mi+1; "mi is smaller than target" } while (lo < hi); "did not find what we wanted" The invariants are: - When entering the loop, 'lo' points at a slot that is never above the target (it could be at the target), 'hi' points at a slot that is guaranteed to be above the target (it can never be at the target). - We find a point 'mi' between 'lo' and 'hi' ('mi' could be the same as 'lo', but never can be as high as 'hi'), and check if 'mi' hits the target. There are three cases: - if it is a hit, we have found what we are looking for; - if it is strictly higher than the target, we set it to 'hi', and repeat the search. - if it is strictly lower than the target, we update 'lo' to one slot after it, because we allow 'lo' to be at the target and 'mi' is known to be below the target. If the loop exits, there is no matching entry. When choosing 'mi', we do not have to take the "middle" but anywhere in between 'lo' and 'hi', as long as lo <= mi < hi is satisfied. When we somehow know that the distance between the target and 'lo' is much shorter than the target and 'hi', we could pick 'mi' that is much closer to 'lo' than (hi+lo)/2, which a conventional binary search would pick. This patch takes advantage of the fact that the SHA-1 is a good hash function, and as long as there are enough entries in the table, we can expect uniform distribution. An entry that begins with for example "deadbeef..." is much likely to appear much later than in the midway of a reasonably populated table. In fact, it can be expected to be near 87% (222/256) from the top of the table. This is a work-in-progress and has switches to allow easier experiments and debugging. Exporting GIT_USE_LOOKUP environment variable enables this code. On my admittedly memory starved machine, with a partial KDE repository (3.0G pack with 95M idx): $ GIT_USE_LOOKUP=t git log -800 --stat HEAD >/dev/null 3.93user 0.16system 0:04.09elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+55588minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -800 --stat HEAD >/dev/null 4.00user 0.15system 0:04.17elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+60258minor)pagefaults 0swaps In the same repository: $ GIT_USE_LOOKUP=t git log -2000 HEAD >/dev/null 0.12user 0.00system 0:00.12elapsed 97%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+4241minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -2000 HEAD >/dev/null 0.05user 0.01system 0:00.07elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+8506minor)pagefaults 0swaps There isn't much time difference, but the number of minor faults seems to show that we are touching much smaller number of pages, which is expected. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-12-29 13:05:47 +03:00
#include "sha1-lookup.h"
#include "bulk-checkin.h"
#include "streaming.h"
#ifndef O_NOATIME
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
#define O_NOATIME 01000000
#else
#define O_NOATIME 0
#endif
#endif
#define SZ_FMT PRIuMAX
static inline uintmax_t sz_fmt(size_t s) { return s; }
const unsigned char null_sha1[20];
/*
* This is meant to hold a *small* number of objects that you would
* want read_sha1_file() to be able to return, but yet you do not want
* to write them into the object store (e.g. a browse-only
* application).
*/
static struct cached_object {
unsigned char sha1[20];
enum object_type type;
void *buf;
unsigned long size;
} *cached_objects;
static int cached_object_nr, cached_object_alloc;
static struct cached_object empty_tree = {
EMPTY_TREE_SHA1_BIN_LITERAL,
OBJ_TREE,
"",
0
};
static struct packed_git *last_found_pack;
static struct cached_object *find_cached_object(const unsigned char *sha1)
{
int i;
struct cached_object *co = cached_objects;
for (i = 0; i < cached_object_nr; i++, co++) {
if (!hashcmp(co->sha1, sha1))
return co;
}
if (!hashcmp(sha1, empty_tree.sha1))
return &empty_tree;
return NULL;
}
rerere: make sure it works even in a workdir attached to a young repository The git-new-workdir script in contrib/ makes a new work tree by sharing many subdirectories of the .git directory with the original repository. When rerere.enabled is set in the original repository, but the user has not encountered any conflicts yet, the original repository may not yet have .git/rr-cache directory. When rerere wants to run in a new work tree created from such a young original repository, it fails to mkdir(2) .git/rr-cache that is a symlink to a yet-to-be-created directory. There are three possible approaches to this: - A naive solution is not to create a symlink in the git-new-workdir script to a directory the original does not have (yet). This is not a solution, as we tend to lazily create subdirectories of .git/, and having rerere.enabled configuration set is a strong indication that the user _wants_ to have this lazy creation to happen; - We could always create .git/rr-cache upon repository creation. This is tempting but will not help people with existing repositories. - Detect this case by seeing that mkdir(2) failed with EEXIST, checking that the path is a symlink, and try running mkdir(2) on the link target. This patch solves the issue by doing the third one. Strictly speaking, this is incomplete. It does not attempt to handle relative symbolic link that points into the original repository, but this is good enough to help people who use contrib/workdir/git-new-workdir script. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-03-11 03:02:50 +03:00
int mkdir_in_gitdir(const char *path)
{
if (mkdir(path, 0777)) {
int saved_errno = errno;
struct stat st;
struct strbuf sb = STRBUF_INIT;
if (errno != EEXIST)
return -1;
/*
* Are we looking at a path in a symlinked worktree
* whose original repository does not yet have it?
* e.g. .git/rr-cache pointing at its original
* repository in which the user hasn't performed any
* conflict resolution yet?
*/
if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
strbuf_readlink(&sb, path, st.st_size) ||
!is_absolute_path(sb.buf) ||
mkdir(sb.buf, 0777)) {
strbuf_release(&sb);
errno = saved_errno;
return -1;
}
strbuf_release(&sb);
}
return adjust_shared_perm(path);
}
int safe_create_leading_directories(char *path)
{
char *pos = path + offset_1st_component(path);
struct stat st;
while (pos) {
pos = strchr(pos, '/');
if (!pos)
break;
while (*++pos == '/')
;
if (!*pos)
break;
*--pos = '\0';
if (!stat(path, &st)) {
/* path exists */
if (!S_ISDIR(st.st_mode)) {
*pos = '/';
return -3;
}
}
else if (mkdir(path, 0777)) {
*pos = '/';
return -1;
}
else if (adjust_shared_perm(path)) {
*pos = '/';
return -2;
}
*pos++ = '/';
}
return 0;
}
int safe_create_leading_directories_const(const char *path)
{
/* path points to cache entries, so xstrdup before messing with it */
char *buf = xstrdup(path);
int result = safe_create_leading_directories(buf);
free(buf);
return result;
}
static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
{
int i;
for (i = 0; i < 20; i++) {
static char hex[] = "0123456789abcdef";
unsigned int val = sha1[i];
char *pos = pathbuf + i*2 + (i > 0);
*pos++ = hex[val >> 4];
*pos = hex[val & 0xf];
}
}
/*
* NOTE! This returns a statically allocated buffer, so you have to be
* careful about using it. Do an "xstrdup()" if you need to save the
* filename.
*
* Also note that this returns the location for creating. Reading
* SHA1 file can happen from any alternate directory listed in the
Rename environment variables. H. Peter Anvin mentioned that using SHA1_whatever as an environment variable name is not nice and we should instead use names starting with "GIT_" prefix to avoid conflicts. Here is what this patch does: * Renames the following environment variables: New name Old Name GIT_AUTHOR_DATE AUTHOR_DATE GIT_AUTHOR_EMAIL AUTHOR_EMAIL GIT_AUTHOR_NAME AUTHOR_NAME GIT_COMMITTER_EMAIL COMMIT_AUTHOR_EMAIL GIT_COMMITTER_NAME COMMIT_AUTHOR_NAME GIT_ALTERNATE_OBJECT_DIRECTORIES SHA1_FILE_DIRECTORIES GIT_OBJECT_DIRECTORY SHA1_FILE_DIRECTORY * Introduces a compatibility macro, gitenv(), which does an getenv() and if it fails calls gitenv_bc(), which in turn picks up the value from old name while giving a warning about using an old name. * Changes all users of the environment variable to fetch environment variable with the new name using gitenv(). * Updates the documentation and scripts shipped with Linus GIT distribution. The transition plan is as follows: * We will keep the backward compatibility list used by gitenv() for now, so the current scripts and user environments continue to work as before. The users will get warnings when they have old name but not new name in their environment to the stderr. * The Porcelain layers should start using new names. However, just in case it ends up calling old Plumbing layer implementation, they should also export old names, taking values from the corresponding new names, during the transition period. * After a transition period, we would drop the compatibility support and drop gitenv(). Revert the callers to directly call getenv() but keep using the new names. The last part is probably optional and the transition duration needs to be set to a reasonable value. Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-05-10 04:57:56 +04:00
* DB_ENVIRONMENT environment variable if it is not found in
* the primary object database.
*/
char *sha1_file_name(const unsigned char *sha1)
{
static char buf[PATH_MAX];
const char *objdir;
int len;
objdir = get_object_directory();
len = strlen(objdir);
/* '/' + sha1(2) + '/' + sha1(38) + '\0' */
if (len + 43 > PATH_MAX)
die("insanely long object directory %s", objdir);
memcpy(buf, objdir, len);
buf[len] = '/';
buf[len+3] = '/';
buf[len+42] = '\0';
fill_sha1_path(buf + len + 1, sha1);
return buf;
}
static char *sha1_get_pack_name(const unsigned char *sha1,
char **name, char **base, const char *which)
{
static const char hex[] = "0123456789abcdef";
char *buf;
int i;
if (!*base) {
const char *sha1_file_directory = get_object_directory();
int len = strlen(sha1_file_directory);
*base = xmalloc(len + 60);
sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s",
sha1_file_directory, which);
*name = *base + len + 11;
}
buf = *name;
for (i = 0; i < 20; i++) {
unsigned int val = *sha1++;
*buf++ = hex[val >> 4];
*buf++ = hex[val & 0xf];
}
return *base;
}
char *sha1_pack_name(const unsigned char *sha1)
{
static char *name, *base;
return sha1_get_pack_name(sha1, &name, &base, "pack");
}
char *sha1_pack_index_name(const unsigned char *sha1)
{
static char *name, *base;
return sha1_get_pack_name(sha1, &name, &base, "idx");
}
struct alternate_object_database *alt_odb_list;
static struct alternate_object_database **alt_odb_tail;
static int git_open_noatime(const char *name);
/*
* Prepare alternate object database registry.
*
* The variable alt_odb_list points at the list of struct
* alternate_object_database. The elements on this list come from
* non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
* environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
* whose contents is similar to that environment variable but can be
* LF separated. Its base points at a statically allocated buffer that
* contains "/the/directory/corresponding/to/.git/objects/...", while
* its name points just after the slash at the end of ".git/objects/"
* in the example above, and has enough space to hold 40-byte hex
* SHA1, an extra slash for the first level indirection, and the
* terminating NUL.
*/
static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth)
{
const char *objdir = get_object_directory();
struct alternate_object_database *ent;
struct alternate_object_database *alt;
int pfxlen, entlen;
struct strbuf pathbuf = STRBUF_INIT;
if (!is_absolute_path(entry) && relative_base) {
strbuf_addstr(&pathbuf, real_path(relative_base));
strbuf_addch(&pathbuf, '/');
}
strbuf_addstr(&pathbuf, entry);
normalize_path_copy(pathbuf.buf, pathbuf.buf);
pfxlen = strlen(pathbuf.buf);
/*
* The trailing slash after the directory name is given by
* this function at the end. Remove duplicates.
*/
while (pfxlen && pathbuf.buf[pfxlen-1] == '/')
pfxlen -= 1;
entlen = pfxlen + 43; /* '/' + 2 hex + '/' + 38 hex + NUL */
ent = xmalloc(sizeof(*ent) + entlen);
memcpy(ent->base, pathbuf.buf, pfxlen);
strbuf_release(&pathbuf);
ent->name = ent->base + pfxlen + 1;
ent->base[pfxlen + 3] = '/';
ent->base[pfxlen] = ent->base[entlen-1] = 0;
/* Detect cases where alternate disappeared */
if (!is_directory(ent->base)) {
error("object directory %s does not exist; "
"check .git/objects/info/alternates.",
ent->base);
free(ent);
return -1;
}
/* Prevent the common mistake of listing the same
* thing twice, or object directory itself.
*/
for (alt = alt_odb_list; alt; alt = alt->next) {
if (!memcmp(ent->base, alt->base, pfxlen)) {
free(ent);
return -1;
}
}
if (!strcmp(ent->base, objdir)) {
free(ent);
return -1;
}
/* add the alternate entry */
*alt_odb_tail = ent;
alt_odb_tail = &(ent->next);
ent->next = NULL;
/* recursively add alternates */
read_info_alternates(ent->base, depth + 1);
ent->base[pfxlen] = '/';
return 0;
}
static void link_alt_odb_entries(const char *alt, int len, int sep,
const char *relative_base, int depth)
{
struct string_list entries = STRING_LIST_INIT_NODUP;
char *alt_copy;
int i;
if (depth > 5) {
error("%s: ignoring alternate object stores, nesting too deep.",
relative_base);
return;
}
alt_copy = xmemdupz(alt, len);
string_list_split_in_place(&entries, alt_copy, sep, -1);
for (i = 0; i < entries.nr; i++) {
const char *entry = entries.items[i].string;
if (entry[0] == '\0' || entry[0] == '#')
continue;
if (!is_absolute_path(entry) && depth) {
error("%s: ignoring relative alternate object store %s",
relative_base, entry);
} else {
link_alt_odb_entry(entry, relative_base, depth);
}
}
string_list_clear(&entries, 0);
free(alt_copy);
}
void read_info_alternates(const char * relative_base, int depth)
{
char *map;
size_t mapsz;
struct stat st;
const char alt_file_name[] = "info/alternates";
/* Given that relative_base is no longer than PATH_MAX,
ensure that "path" has enough space to append "/", the
file name, "info/alternates", and a trailing NUL. */
char path[PATH_MAX + 1 + sizeof alt_file_name];
int fd;
sprintf(path, "%s/%s", relative_base, alt_file_name);
fd = git_open_noatime(path);
if (fd < 0)
return;
if (fstat(fd, &st) || (st.st_size == 0)) {
close(fd);
return;
}
mapsz = xsize_t(st.st_size);
map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
munmap(map, mapsz);
}
void add_to_alternates_file(const char *reference)
{
struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR);
char *alt = mkpath("%s\n", reference);
write_or_die(fd, alt, strlen(alt));
if (commit_lock_file(lock))
die("could not close alternates file");
if (alt_odb_tail)
link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
}
void foreach_alt_odb(alt_odb_fn fn, void *cb)
{
struct alternate_object_database *ent;
prepare_alt_odb();
for (ent = alt_odb_list; ent; ent = ent->next)
if (fn(ent, cb))
return;
}
void prepare_alt_odb(void)
{
const char *alt;
if (alt_odb_tail)
return;
alt = getenv(ALTERNATE_DB_ENVIRONMENT);
if (!alt) alt = "";
alt_odb_tail = &alt_odb_list;
link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
read_info_alternates(get_object_directory(), 0);
}
static int has_loose_object_local(const unsigned char *sha1)
{
char *name = sha1_file_name(sha1);
return !access(name, F_OK);
}
int has_loose_object_nonlocal(const unsigned char *sha1)
{
struct alternate_object_database *alt;
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
fill_sha1_path(alt->name, sha1);
if (!access(alt->base, F_OK))
return 1;
}
return 0;
}
static int has_loose_object(const unsigned char *sha1)
{
return has_loose_object_local(sha1) ||
has_loose_object_nonlocal(sha1);
}
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
static unsigned int pack_used_ctr;
static unsigned int pack_mmap_calls;
static unsigned int peak_pack_open_windows;
static unsigned int pack_open_windows;
static unsigned int pack_open_fds;
static unsigned int pack_max_fds;
static size_t peak_pack_mapped;
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
static size_t pack_mapped;
struct packed_git *packed_git;
void pack_report(void)
{
fprintf(stderr,
"pack_report: getpagesize() = %10" SZ_FMT "\n"
"pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
"pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
sz_fmt(getpagesize()),
sz_fmt(packed_git_window_size),
sz_fmt(packed_git_limit));
fprintf(stderr,
"pack_report: pack_used_ctr = %10u\n"
"pack_report: pack_mmap_calls = %10u\n"
"pack_report: pack_open_windows = %10u / %10u\n"
"pack_report: pack_mapped = "
"%10" SZ_FMT " / %10" SZ_FMT "\n",
pack_used_ctr,
pack_mmap_calls,
pack_open_windows, peak_pack_open_windows,
sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
}
static int check_packed_git_idx(const char *path, struct packed_git *p)
{
void *idx_map;
struct pack_idx_header *hdr;
size_t idx_size;
uint32_t version, nr, i, *index;
int fd = git_open_noatime(path);
struct stat st;
if (fd < 0)
return -1;
if (fstat(fd, &st)) {
close(fd);
return -1;
}
idx_size = xsize_t(st.st_size);
if (idx_size < 4 * 256 + 20 + 20) {
close(fd);
return error("index file %s is too small", path);
}
idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
hdr = idx_map;
if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
version = ntohl(hdr->idx_version);
if (version < 2 || version > 2) {
munmap(idx_map, idx_size);
return error("index file %s is version %"PRIu32
" and is not supported by this binary"
" (try upgrading GIT to a newer version)",
path, version);
}
} else
version = 1;
nr = 0;
index = idx_map;
if (version > 1)
index += 2; /* skip index header */
for (i = 0; i < 256; i++) {
uint32_t n = ntohl(index[i]);
if (n < nr) {
munmap(idx_map, idx_size);
return error("non-monotonic index %s", path);
}
nr = n;
}
if (version == 1) {
/*
* Total size:
* - 256 index entries 4 bytes each
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
* - 20-byte SHA1 of the packfile
* - 20-byte SHA1 file checksum
*/
if (idx_size != 4*256 + nr * 24 + 20 + 20) {
munmap(idx_map, idx_size);
return error("wrong index v1 file size in %s", path);
}
} else if (version == 2) {
/*
* Minimum size:
* - 8 bytes of header
* - 256 index entries 4 bytes each
* - 20-byte sha1 entry * nr
* - 4-byte crc entry * nr
* - 4-byte offset entry * nr
* - 20-byte SHA1 of the packfile
* - 20-byte SHA1 file checksum
* And after the 4-byte offset table might be a
* variable sized table containing 8-byte entries
* for offsets larger than 2^31.
*/
unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
unsigned long max_size = min_size;
if (nr)
max_size += (nr - 1)*8;
if (idx_size < min_size || idx_size > max_size) {
munmap(idx_map, idx_size);
return error("wrong index v2 file size in %s", path);
}
if (idx_size != min_size &&
/*
* make sure we can deal with large pack offsets.
* 31-bit signed offset won't be enough, neither
* 32-bit unsigned one will be.
*/
(sizeof(off_t) <= 4)) {
munmap(idx_map, idx_size);
return error("pack too large for current definition of off_t in %s", path);
}
}
p->index_version = version;
p->index_data = idx_map;
p->index_size = idx_size;
p->num_objects = nr;
return 0;
}
int open_pack_index(struct packed_git *p)
{
char *idx_name;
int ret;
if (p->index_data)
return 0;
idx_name = xstrdup(p->pack_name);
strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
ret = check_packed_git_idx(idx_name, p);
free(idx_name);
return ret;
}
static void scan_windows(struct packed_git *p,
struct packed_git **lru_p,
struct pack_window **lru_w,
struct pack_window **lru_l)
{
struct pack_window *w, *w_l;
for (w_l = NULL, w = p->windows; w; w = w->next) {
if (!w->inuse_cnt) {
if (!*lru_w || w->last_used < (*lru_w)->last_used) {
*lru_p = p;
*lru_w = w;
*lru_l = w_l;
}
}
w_l = w;
}
}
Actually handle some-low memory conditions Tim Ansell discovered his Debian server didn't permit git-daemon to use as much memory as it needed to handle cloning a project with a 128 MiB packfile. Filtering the strace provided by Tim of the rev-list child showed this gem of a sequence: open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...> <... open resumed> ) = 5 OK, so the packfile is fd 5... mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...> <... mmap2 resumed> ) = 0xb5e2d000 and we mapped one 32 MiB window from it at position 0... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory) And we asked for another window further into the file. But got denied. In Tim's case this was due to a resource limit on the git-daemon process, and its children. Now where are we in the code? We're down inside use_pack(), after we have called unuse_one_window() enough times to make sure we stay within our allowed maximum window size. However since we didn't unmap the prior window at 0xb5e2d000 we aren't exceeding the current limit (which probably was just the defaults). But we're actually down inside xmmap()... So we release the window we do have (by calling release_pack_memory), assuming there is some memory pressure... munmap(0xb5e2d000, 33554432 <unfinished ...> <... munmap resumed> ) = 0 close(5 <unfinished ...> <... close resumed> ) = 0 And that was the last window in this packfile. So we closed it. Way to go us. Our xmmap did not expect release_pack_memory to close the fd its about to map... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 EBADF (Bad file descriptor) And so the Linux kernel happily tells us f' off. write(2, "fatal: ", 7 <unfinished ...> <... write resumed> ) = 7 write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...> <... write resumed> ) = 47 And we report the bad file descriptor error, and not the ENOMEM, and die, claiming we are out of memory. But actually that mmap should have succeeded, as we had enough memory for that window, seeing as how we released the prior one. Originally when I developed the sliding window mmap feature I had this exact same bug in fast-import, and I dealt with it by handing in the struct packed_git* we want to open the new window for, as the caller wasn't prepared to reopen the packfile if unuse_one_window closed it. The same is true here from xmmap, but the caller doesn't have the struct packed_git* handy. So I'm using the file descriptor instead to perform the same test. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
static int unuse_one_window(struct packed_git *current, int keep_fd)
{
struct packed_git *p, *lru_p = NULL;
struct pack_window *lru_w = NULL, *lru_l = NULL;
if (current)
scan_windows(current, &lru_p, &lru_w, &lru_l);
for (p = packed_git; p; p = p->next)
scan_windows(p, &lru_p, &lru_w, &lru_l);
if (lru_p) {
munmap(lru_w->base, lru_w->len);
pack_mapped -= lru_w->len;
if (lru_l)
lru_l->next = lru_w->next;
else {
lru_p->windows = lru_w->next;
if (!lru_p->windows && lru_p->pack_fd != -1
&& lru_p->pack_fd != keep_fd) {
close(lru_p->pack_fd);
pack_open_fds--;
lru_p->pack_fd = -1;
}
}
free(lru_w);
pack_open_windows--;
return 1;
}
return 0;
}
Actually handle some-low memory conditions Tim Ansell discovered his Debian server didn't permit git-daemon to use as much memory as it needed to handle cloning a project with a 128 MiB packfile. Filtering the strace provided by Tim of the rev-list child showed this gem of a sequence: open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...> <... open resumed> ) = 5 OK, so the packfile is fd 5... mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...> <... mmap2 resumed> ) = 0xb5e2d000 and we mapped one 32 MiB window from it at position 0... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory) And we asked for another window further into the file. But got denied. In Tim's case this was due to a resource limit on the git-daemon process, and its children. Now where are we in the code? We're down inside use_pack(), after we have called unuse_one_window() enough times to make sure we stay within our allowed maximum window size. However since we didn't unmap the prior window at 0xb5e2d000 we aren't exceeding the current limit (which probably was just the defaults). But we're actually down inside xmmap()... So we release the window we do have (by calling release_pack_memory), assuming there is some memory pressure... munmap(0xb5e2d000, 33554432 <unfinished ...> <... munmap resumed> ) = 0 close(5 <unfinished ...> <... close resumed> ) = 0 And that was the last window in this packfile. So we closed it. Way to go us. Our xmmap did not expect release_pack_memory to close the fd its about to map... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 EBADF (Bad file descriptor) And so the Linux kernel happily tells us f' off. write(2, "fatal: ", 7 <unfinished ...> <... write resumed> ) = 7 write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...> <... write resumed> ) = 47 And we report the bad file descriptor error, and not the ENOMEM, and die, claiming we are out of memory. But actually that mmap should have succeeded, as we had enough memory for that window, seeing as how we released the prior one. Originally when I developed the sliding window mmap feature I had this exact same bug in fast-import, and I dealt with it by handing in the struct packed_git* we want to open the new window for, as the caller wasn't prepared to reopen the packfile if unuse_one_window closed it. The same is true here from xmmap, but the caller doesn't have the struct packed_git* handy. So I'm using the file descriptor instead to perform the same test. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
void release_pack_memory(size_t need, int fd)
{
size_t cur = pack_mapped;
Actually handle some-low memory conditions Tim Ansell discovered his Debian server didn't permit git-daemon to use as much memory as it needed to handle cloning a project with a 128 MiB packfile. Filtering the strace provided by Tim of the rev-list child showed this gem of a sequence: open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...> <... open resumed> ) = 5 OK, so the packfile is fd 5... mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...> <... mmap2 resumed> ) = 0xb5e2d000 and we mapped one 32 MiB window from it at position 0... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory) And we asked for another window further into the file. But got denied. In Tim's case this was due to a resource limit on the git-daemon process, and its children. Now where are we in the code? We're down inside use_pack(), after we have called unuse_one_window() enough times to make sure we stay within our allowed maximum window size. However since we didn't unmap the prior window at 0xb5e2d000 we aren't exceeding the current limit (which probably was just the defaults). But we're actually down inside xmmap()... So we release the window we do have (by calling release_pack_memory), assuming there is some memory pressure... munmap(0xb5e2d000, 33554432 <unfinished ...> <... munmap resumed> ) = 0 close(5 <unfinished ...> <... close resumed> ) = 0 And that was the last window in this packfile. So we closed it. Way to go us. Our xmmap did not expect release_pack_memory to close the fd its about to map... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 EBADF (Bad file descriptor) And so the Linux kernel happily tells us f' off. write(2, "fatal: ", 7 <unfinished ...> <... write resumed> ) = 7 write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...> <... write resumed> ) = 47 And we report the bad file descriptor error, and not the ENOMEM, and die, claiming we are out of memory. But actually that mmap should have succeeded, as we had enough memory for that window, seeing as how we released the prior one. Originally when I developed the sliding window mmap feature I had this exact same bug in fast-import, and I dealt with it by handing in the struct packed_git* we want to open the new window for, as the caller wasn't prepared to reopen the packfile if unuse_one_window closed it. The same is true here from xmmap, but the caller doesn't have the struct packed_git* handy. So I'm using the file descriptor instead to perform the same test. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
; /* nothing */
}
void *xmmap(void *start, size_t length,
int prot, int flags, int fd, off_t offset)
{
void *ret = mmap(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED) {
if (!length)
return NULL;
release_pack_memory(length, fd);
ret = mmap(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED)
die_errno("Out of memory? mmap failed");
}
return ret;
}
Fix random fast-import errors when compiled with NO_MMAP fast-import was relying on the fact that on most systems mmap() and write() are synchronized by the filesystem's buffer cache. We were relying on the ability to mmap() 20 bytes beyond the current end of the file, then later fill in those bytes with a future write() call, then read them through the previously obtained mmap() address. This isn't always true with some implementations of NFS, but it is especially not true with our NO_MMAP=YesPlease build time option used on some platforms. If fast-import was built with NO_MMAP=YesPlease we used the malloc()+pread() emulation and the subsequent write() call does not update the trailing 20 bytes of a previously obtained "mmap()" (aka malloc'd) address. Under NO_MMAP that behavior causes unpack_entry() in sha1_file.c to be unable to read an object header (or data) that has been unlucky enough to be written to the packfile at a location such that it is in the trailing 20 bytes of a window previously opened on that same packfile. This bug has gone unnoticed for a very long time as it is highly data dependent. Not only does the object have to be placed at the right position, but it also needs to be positioned behind some other object that has been accessed due to a branch cache invalidation. In other words the stars had to align just right, and if you did run into this bug you probably should also have purchased a lottery ticket. Fortunately the workaround is a lot easier than the bug explanation. Before we allow unpack_entry() to read data from a pack window that has also (possibly) been modified through write() we force all existing windows on that packfile to be closed. By closing the windows we ensure that any new access via the emulated mmap() will reread the packfile, updating to the current file content. This comes at a slight performance degredation as we cannot reuse previously cached windows when we update the packfile. But it is a fairly minor difference as the window closes happen at only two points: - When the packfile is finalized and its .idx is generated: At this stage we are getting ready to update the refs and any data access into the packfile is going to be random, and is going after only the branch tips (to ensure they are valid). Our existing windows (if any) are not likely to be positioned at useful locations to access those final tip commits so we probably were closing them before anyway. - When the branch cache missed and we need to reload: At this point fast-import is getting change commands for the next commit and it needs to go re-read a tree object it previously had written out to the packfile. What windows we had (if any) are not likely to cover the tree in question so we probably were closing them before anyway. We do try to avoid unnecessarily closing windows in the second case by checking to see if the packfile size has increased since the last time we called unpack_entry() on that packfile. If the size has not changed then we have not written additional data, and any existing window is still vaild. This nicely handles the cases where fast-import is going through a branch cache reload and needs to read many trees at once. During such an event we are not likely to be updating the packfile so we do not cycle the windows between reads. With this change in place t9301-fast-export.sh (which was broken by c3b0dec509fe136c5417422f31898b5a4e2d5e02) finally works again. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-01-18 06:57:00 +03:00
void close_pack_windows(struct packed_git *p)
{
while (p->windows) {
struct pack_window *w = p->windows;
if (w->inuse_cnt)
die("pack '%s' still has open windows to it",
p->pack_name);
munmap(w->base, w->len);
pack_mapped -= w->len;
pack_open_windows--;
p->windows = w->next;
free(w);
}
}
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
void unuse_pack(struct pack_window **w_cursor)
{
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window *w = *w_cursor;
if (w) {
w->inuse_cnt--;
*w_cursor = NULL;
}
}
void close_pack_index(struct packed_git *p)
{
if (p->index_data) {
munmap((void *)p->index_data, p->index_size);
p->index_data = NULL;
}
}
/*
* This is used by git-repack in case a newly created pack happens to
* contain the same set of objects as an existing one. In that case
* the resulting file might be different even if its name would be the
* same. It is best to close any reference to the old pack before it is
* replaced on disk. Of course no index pointers nor windows for given pack
* must subsist at this point. If ever objects from this pack are requested
* again, the new version of the pack will be reinitialized through
* reprepare_packed_git().
*/
void free_pack_by_name(const char *pack_name)
{
struct packed_git *p, **pp = &packed_git;
while (*pp) {
p = *pp;
if (strcmp(pack_name, p->pack_name) == 0) {
clear_delta_base_cache();
close_pack_windows(p);
if (p->pack_fd != -1) {
close(p->pack_fd);
pack_open_fds--;
}
close_pack_index(p);
free(p->bad_object_sha1);
*pp = p->next;
if (last_found_pack == p)
last_found_pack = NULL;
free(p);
return;
}
pp = &p->next;
}
}
static unsigned int get_max_fd_limit(void)
{
#ifdef RLIMIT_NOFILE
struct rlimit lim;
if (getrlimit(RLIMIT_NOFILE, &lim))
die_errno("cannot get RLIMIT_NOFILE");
return lim.rlim_cur;
#elif defined(_SC_OPEN_MAX)
return sysconf(_SC_OPEN_MAX);
#elif defined(OPEN_MAX)
return OPEN_MAX;
#else
return 1; /* see the caller ;-) */
#endif
}
/*
* Do not call this directly as this leaks p->pack_fd on error return;
* call open_packed_git() instead.
*/
static int open_packed_git_1(struct packed_git *p)
{
struct stat st;
struct pack_header hdr;
unsigned char sha1[20];
unsigned char *idx_sha1;
long fd_flag;
if (!p->index_data && open_pack_index(p))
return error("packfile %s index unavailable", p->pack_name);
if (!pack_max_fds) {
unsigned int max_fds = get_max_fd_limit();
/* Save 3 for stdin/stdout/stderr, 22 for work */
if (25 < max_fds)
pack_max_fds = max_fds - 25;
else
pack_max_fds = 1;
}
while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))
; /* nothing */
p->pack_fd = git_open_noatime(p->pack_name);
if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
return -1;
pack_open_fds++;
/* If we created the struct before we had the pack we lack size. */
if (!p->pack_size) {
if (!S_ISREG(st.st_mode))
return error("packfile %s not a regular file", p->pack_name);
p->pack_size = st.st_size;
} else if (p->pack_size != st.st_size)
return error("packfile %s size changed", p->pack_name);
/* We leave these file descriptors open with sliding mmap;
* there is no point keeping them open across exec(), though.
*/
fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
if (fd_flag < 0)
return error("cannot determine file descriptor flags");
fd_flag |= FD_CLOEXEC;
if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
return error("cannot set FD_CLOEXEC");
/* Verify we recognize this pack file format. */
if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
return error("file %s is far too short to be a packfile", p->pack_name);
if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
return error("file %s is not a GIT packfile", p->pack_name);
if (!pack_version_ok(hdr.hdr_version))
return error("packfile %s is version %"PRIu32" and not"
" supported (try upgrading GIT to a newer version)",
p->pack_name, ntohl(hdr.hdr_version));
/* Verify the pack matches its index. */
if (p->num_objects != ntohl(hdr.hdr_entries))
return error("packfile %s claims to have %"PRIu32" objects"
" while index indicates %"PRIu32" objects",
p->pack_name, ntohl(hdr.hdr_entries),
p->num_objects);
if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
return error("end of packfile %s is unavailable", p->pack_name);
if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
return error("packfile %s signature is unavailable", p->pack_name);
idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
if (hashcmp(sha1, idx_sha1))
return error("packfile %s does not match index", p->pack_name);
return 0;
}
static int open_packed_git(struct packed_git *p)
{
if (!open_packed_git_1(p))
return 0;
if (p->pack_fd != -1) {
close(p->pack_fd);
pack_open_fds--;
p->pack_fd = -1;
}
return -1;
}
static int in_window(struct pack_window *win, off_t offset)
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
{
/* We must promise at least 20 bytes (one hash) after the
* offset is available from this window, otherwise the offset
* is not actually in this window and a different window (which
* has that one hash excess) must be used. This is to support
* the object header and delta base parsing routines below.
*/
off_t win_off = win->offset;
return win_off <= offset
&& (offset + 20) <= (win_off + win->len);
}
unsigned char *use_pack(struct packed_git *p,
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window **w_cursor,
off_t offset,
2011-06-10 22:52:15 +04:00
unsigned long *left)
{
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
struct pack_window *win = *w_cursor;
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
/* Since packfiles end in a hash of their content and it's
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
* pointless to ask for an offset into the middle of that
* hash, and the in_window function above wouldn't match
* don't allow an offset too close to the end of the file.
*/
if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
die("packfile %s cannot be accessed", p->pack_name);
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
if (offset > (p->pack_size - 20))
die("offset beyond end of packfile (truncated pack?)");
if (!win || !in_window(win, offset)) {
if (win)
win->inuse_cnt--;
for (win = p->windows; win; win = win->next) {
if (in_window(win, offset))
break;
}
if (!win) {
size_t window_align = packed_git_window_size / 2;
off_t len;
if (p->pack_fd == -1 && open_packed_git(p))
die("packfile %s cannot be accessed", p->pack_name);
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
win = xcalloc(1, sizeof(*win));
win->offset = (offset / window_align) * window_align;
len = p->pack_size - win->offset;
if (len > packed_git_window_size)
len = packed_git_window_size;
win->len = (size_t)len;
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
pack_mapped += win->len;
while (packed_git_limit < pack_mapped
Actually handle some-low memory conditions Tim Ansell discovered his Debian server didn't permit git-daemon to use as much memory as it needed to handle cloning a project with a 128 MiB packfile. Filtering the strace provided by Tim of the rev-list child showed this gem of a sequence: open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...> <... open resumed> ) = 5 OK, so the packfile is fd 5... mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...> <... mmap2 resumed> ) = 0xb5e2d000 and we mapped one 32 MiB window from it at position 0... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory) And we asked for another window further into the file. But got denied. In Tim's case this was due to a resource limit on the git-daemon process, and its children. Now where are we in the code? We're down inside use_pack(), after we have called unuse_one_window() enough times to make sure we stay within our allowed maximum window size. However since we didn't unmap the prior window at 0xb5e2d000 we aren't exceeding the current limit (which probably was just the defaults). But we're actually down inside xmmap()... So we release the window we do have (by calling release_pack_memory), assuming there is some memory pressure... munmap(0xb5e2d000, 33554432 <unfinished ...> <... munmap resumed> ) = 0 close(5 <unfinished ...> <... close resumed> ) = 0 And that was the last window in this packfile. So we closed it. Way to go us. Our xmmap did not expect release_pack_memory to close the fd its about to map... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 EBADF (Bad file descriptor) And so the Linux kernel happily tells us f' off. write(2, "fatal: ", 7 <unfinished ...> <... write resumed> ) = 7 write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...> <... write resumed> ) = 47 And we report the bad file descriptor error, and not the ENOMEM, and die, claiming we are out of memory. But actually that mmap should have succeeded, as we had enough memory for that window, seeing as how we released the prior one. Originally when I developed the sliding window mmap feature I had this exact same bug in fast-import, and I dealt with it by handing in the struct packed_git* we want to open the new window for, as the caller wasn't prepared to reopen the packfile if unuse_one_window closed it. The same is true here from xmmap, but the caller doesn't have the struct packed_git* handy. So I'm using the file descriptor instead to perform the same test. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
&& unuse_one_window(p, p->pack_fd))
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
; /* nothing */
win->base = xmmap(NULL, win->len,
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
PROT_READ, MAP_PRIVATE,
p->pack_fd, win->offset);
if (win->base == MAP_FAILED)
die("packfile %s cannot be mapped: %s",
p->pack_name,
strerror(errno));
if (!win->offset && win->len == p->pack_size
&& !p->do_not_close) {
close(p->pack_fd);
pack_open_fds--;
p->pack_fd = -1;
}
pack_mmap_calls++;
pack_open_windows++;
if (pack_mapped > peak_pack_mapped)
peak_pack_mapped = pack_mapped;
if (pack_open_windows > peak_pack_open_windows)
peak_pack_open_windows = pack_open_windows;
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
win->next = p->windows;
p->windows = win;
}
}
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
if (win != *w_cursor) {
win->last_used = pack_used_ctr++;
win->inuse_cnt++;
*w_cursor = win;
}
Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:28 +03:00
offset -= win->offset;
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
if (left)
*left = win->len - xsize_t(offset);
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
return win->base + offset;
}
static struct packed_git *alloc_packed_git(int extra)
{
struct packed_git *p = xmalloc(sizeof(*p) + extra);
memset(p, 0, sizeof(*p));
p->pack_fd = -1;
return p;
}
static void try_to_free_pack_memory(size_t size)
{
release_pack_memory(size, -1);
}
struct packed_git *add_packed_git(const char *path, int path_len, int local)
{
static int have_set_try_to_free_routine;
struct stat st;
struct packed_git *p = alloc_packed_git(path_len + 2);
if (!have_set_try_to_free_routine) {
have_set_try_to_free_routine = 1;
set_try_to_free_routine(try_to_free_pack_memory);
}
/*
* Make sure a corresponding .pack file exists and that
* the index looks sane.
*/
path_len -= strlen(".idx");
if (path_len < 1) {
free(p);
return NULL;
}
memcpy(p->pack_name, path, path_len);
strcpy(p->pack_name + path_len, ".keep");
if (!access(p->pack_name, F_OK))
p->pack_keep = 1;
strcpy(p->pack_name + path_len, ".pack");
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
free(p);
return NULL;
}
/* ok, it looks sane as far as we can check without
* actually mapping the pack file.
*/
p->pack_size = st.st_size;
p->pack_local = local;
p->mtime = st.st_mtime;
if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
hashclr(p->sha1);
return p;
}
struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
{
const char *path = sha1_pack_name(sha1);
struct packed_git *p = alloc_packed_git(strlen(path) + 1);
strcpy(p->pack_name, path);
hashcpy(p->sha1, sha1);
if (check_packed_git_idx(idx_path, p)) {
free(p);
return NULL;
}
return p;
}
void install_packed_git(struct packed_git *pack)
{
if (pack->pack_fd != -1)
pack_open_fds++;
pack->next = packed_git;
packed_git = pack;
}
static void prepare_packed_git_one(char *objdir, int local)
{
/* Ensure that this buffer is large enough so that we can
append "/pack/" without clobbering the stack even if
strlen(objdir) were PATH_MAX. */
char path[PATH_MAX + 1 + 4 + 1 + 1];
int len;
DIR *dir;
struct dirent *de;
sprintf(path, "%s/pack", objdir);
len = strlen(path);
dir = opendir(path);
if (!dir) {
if (errno != ENOENT)
error("unable to open object pack directory: %s: %s",
path, strerror(errno));
return;
}
path[len++] = '/';
while ((de = readdir(dir)) != NULL) {
int namelen = strlen(de->d_name);
struct packed_git *p;
if (!has_extension(de->d_name, ".idx"))
continue;
if (len + namelen + 1 > sizeof(path))
continue;
/* Don't reopen a pack we already have. */
strcpy(path + len, de->d_name);
for (p = packed_git; p; p = p->next) {
if (!memcmp(path, p->pack_name, len + namelen - 4))
break;
}
if (p)
continue;
/* See if it really is a valid .idx file with corresponding
* .pack file that we can map.
*/
p = add_packed_git(path, len + namelen, local);
if (!p)
continue;
install_packed_git(p);
}
closedir(dir);
}
static int sort_pack(const void *a_, const void *b_)
{
struct packed_git *a = *((struct packed_git **)a_);
struct packed_git *b = *((struct packed_git **)b_);
int st;
/*
* Local packs tend to contain objects specific to our
* variant of the project than remote ones. In addition,
* remote ones could be on a network mounted filesystem.
* Favor local ones for these reasons.
*/
st = a->pack_local - b->pack_local;
if (st)
return -st;
/*
* Younger packs tend to contain more recent objects,
* and more recent objects tend to get accessed more
* often.
*/
if (a->mtime < b->mtime)
return 1;
else if (a->mtime == b->mtime)
return 0;
return -1;
}
static void rearrange_packed_git(void)
{
struct packed_git **ary, *p;
int i, n;
for (n = 0, p = packed_git; p; p = p->next)
n++;
if (n < 2)
return;
/* prepare an array of packed_git for easier sorting */
ary = xcalloc(n, sizeof(struct packed_git *));
for (n = 0, p = packed_git; p; p = p->next)
ary[n++] = p;
qsort(ary, n, sizeof(struct packed_git *), sort_pack);
/* link them back again */
for (i = 0; i < n - 1; i++)
ary[i]->next = ary[i + 1];
ary[n - 1]->next = NULL;
packed_git = ary[0];
free(ary);
}
static int prepare_packed_git_run_once = 0;
void prepare_packed_git(void)
{
struct alternate_object_database *alt;
if (prepare_packed_git_run_once)
return;
prepare_packed_git_one(get_object_directory(), 1);
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
alt->name[-1] = 0;
prepare_packed_git_one(alt->base, 0);
alt->name[-1] = '/';
}
rearrange_packed_git();
prepare_packed_git_run_once = 1;
}
void reprepare_packed_git(void)
{
discard_revindex();
prepare_packed_git_run_once = 0;
prepare_packed_git();
}
static void mark_bad_packed_object(struct packed_git *p,
const unsigned char *sha1)
{
unsigned i;
for (i = 0; i < p->num_bad_objects; i++)
if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
return;
p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1));
hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1);
p->num_bad_objects++;
}
static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
{
struct packed_git *p;
unsigned i;
for (p = packed_git; p; p = p->next)
for (i = 0; i < p->num_bad_objects; i++)
if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
return p;
return NULL;
}
/*
* With an in-core object data in "map", rehash it to make sure the
* object name actually matches "sha1" to detect object corruption.
* With "map" == NULL, try reading the object named with "sha1" using
* the streaming interface and rehash it to do the same.
*/
int check_sha1_signature(const unsigned char *sha1, void *map,
unsigned long size, const char *type)
{
unsigned char real_sha1[20];
enum object_type obj_type;
struct git_istream *st;
git_SHA_CTX c;
char hdr[32];
int hdrlen;
if (map) {
hash_sha1_file(map, size, type, real_sha1);
return hashcmp(sha1, real_sha1) ? -1 : 0;
}
st = open_istream(sha1, &obj_type, &size, NULL);
if (!st)
return -1;
/* Generate the header */
hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1;
/* Sha1.. */
git_SHA1_Init(&c);
git_SHA1_Update(&c, hdr, hdrlen);
for (;;) {
char buf[1024 * 16];
ssize_t readlen = read_istream(st, buf, sizeof(buf));
if (!readlen)
break;
git_SHA1_Update(&c, buf, readlen);
}
git_SHA1_Final(real_sha1, &c);
close_istream(st);
return hashcmp(sha1, real_sha1) ? -1 : 0;
}
static int git_open_noatime(const char *name)
{
static int sha1_file_open_flag = O_NOATIME;
for (;;) {
int fd = open(name, O_RDONLY | sha1_file_open_flag);
if (fd >= 0)
return fd;
/* Might the failure be due to O_NOATIME? */
if (errno != ENOENT && sha1_file_open_flag) {
sha1_file_open_flag = 0;
continue;
}
return -1;
}
}
static int open_sha1_file(const unsigned char *sha1)
{
int fd;
char *name = sha1_file_name(sha1);
struct alternate_object_database *alt;
fd = git_open_noatime(name);
if (fd >= 0)
return fd;
prepare_alt_odb();
errno = ENOENT;
for (alt = alt_odb_list; alt; alt = alt->next) {
name = alt->name;
fill_sha1_path(name, sha1);
fd = git_open_noatime(alt->base);
if (fd >= 0)
return fd;
}
return -1;
}
void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
{
void *map;
int fd;
fd = open_sha1_file(sha1);
map = NULL;
if (fd >= 0) {
struct stat st;
if (!fstat(fd, &st)) {
*size = xsize_t(st.st_size);
if (!*size) {
/* mmap() is forbidden on empty files */
error("object file %s is empty", sha1_file_name(sha1));
return NULL;
}
map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
}
close(fd);
}
return map;
}
/*
* There used to be a second loose object header format which
* was meant to mimic the in-pack format, allowing for direct
* copy of the object data. This format turned up not to be
* really worth it and we no longer write loose objects in that
* format.
*/
static int experimental_loose_object(unsigned char *map)
{
unsigned int word;
/*
Tolerate zlib deflation with window size < 32Kb Git currently reports loose objects as 'corrupt' if they've been deflated using a window size less than 32Kb, because the experimental_loose_object() function doesn't recognise the header byte as a zlib header. This patch makes the function tolerant of all valid window sizes (15-bit to 8-bit) - but doesn't sacrifice it's accuracy in distingushing the standard loose-object format from the experimental (now abandoned) format. On memory constrained systems zlib may use a much smaller window size - working on Agit, I found that Android uses a 4KB window; giving a header byte of 0x48, not 0x78. Consequently all loose objects generated appear 'corrupt', which is why Agit is a read-only Git client at this time - I don't want my client to generate Git repos that other clients treat as broken :( This patch makes Git tolerant of different deflate settings - it might appear that it changes experimental_loose_object() to the point where it could incorrectly identify the experimental format as the standard one, but the two criteria (bitmask & checksum) can only give a false result for an experimental object where both of the following are true: 1) object size is exactly 8 bytes when uncompressed (bitmask) 2) [single-byte in-pack git type&size header] * 256 + [1st byte of the following zlib header] % 31 = 0 (checksum) As it happens, for all possible combinations of valid object type (1-4) and window bits (0-7), the only time when the checksum will be divisible by 31 is for 0x1838 - ie object type *1*, a Commit - which, due the fields all Commit objects must contain, could never be as small as 8 bytes in size. Given this, the combination of the two criteria (bitmask & checksum) always correctly determines the buffer format, and is more tolerant than the previous version. The alternative to this patch is simply removing support for the experimental format, which I am also totally cool with. References: Android uses a 4KB window for deflation: http://android.git.kernel.org/?p=platform/libcore.git;a=blob;f=luni/src/main/native/java_util_zip_Deflater.cpp;h=c0b2feff196e63a7b85d97cf9ae5bb2583409c28;hb=refs/heads/gingerbread#l53 Code snippet searching for false positives with the zlib checksum: https://gist.github.com/1118177 Signed-off-by: Roberto Tyley <roberto.tyley@guardian.co.uk> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-07 22:46:13 +04:00
* We must determine if the buffer contains the standard
* zlib-deflated stream or the experimental format based
* on the in-pack object format. Compare the header byte
* for each format:
*
* RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7
* Experimental pack-based : Stttssss : ttt = 1,2,3,4
*
* If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be
* in standard loose-object format, UNLESS it is a Git-pack
* format object *exactly* 8 bytes in size when inflated.
*
* However, RFC1950 also specifies that the 1st 16-bit word
* must be divisible by 31 - this checksum tells us our buffer
* is in the standard format, giving a false positive only if
* the 1st word of the Git-pack format object happens to be
* divisible by 31, ie:
* ((byte0 * 256) + byte1) % 31 = 0
* => 0ttt10000www1000 % 31 = 0
*
* As it happens, this case can only arise for www=3 & ttt=1
* - ie, a Commit object, which would have to be 8 bytes in
* size. As no Commit can be that small, we find that the
* combination of these two criteria (bitmask & checksum)
* can always correctly determine the buffer format.
*/
word = (map[0] << 8) + map[1];
Tolerate zlib deflation with window size < 32Kb Git currently reports loose objects as 'corrupt' if they've been deflated using a window size less than 32Kb, because the experimental_loose_object() function doesn't recognise the header byte as a zlib header. This patch makes the function tolerant of all valid window sizes (15-bit to 8-bit) - but doesn't sacrifice it's accuracy in distingushing the standard loose-object format from the experimental (now abandoned) format. On memory constrained systems zlib may use a much smaller window size - working on Agit, I found that Android uses a 4KB window; giving a header byte of 0x48, not 0x78. Consequently all loose objects generated appear 'corrupt', which is why Agit is a read-only Git client at this time - I don't want my client to generate Git repos that other clients treat as broken :( This patch makes Git tolerant of different deflate settings - it might appear that it changes experimental_loose_object() to the point where it could incorrectly identify the experimental format as the standard one, but the two criteria (bitmask & checksum) can only give a false result for an experimental object where both of the following are true: 1) object size is exactly 8 bytes when uncompressed (bitmask) 2) [single-byte in-pack git type&size header] * 256 + [1st byte of the following zlib header] % 31 = 0 (checksum) As it happens, for all possible combinations of valid object type (1-4) and window bits (0-7), the only time when the checksum will be divisible by 31 is for 0x1838 - ie object type *1*, a Commit - which, due the fields all Commit objects must contain, could never be as small as 8 bytes in size. Given this, the combination of the two criteria (bitmask & checksum) always correctly determines the buffer format, and is more tolerant than the previous version. The alternative to this patch is simply removing support for the experimental format, which I am also totally cool with. References: Android uses a 4KB window for deflation: http://android.git.kernel.org/?p=platform/libcore.git;a=blob;f=luni/src/main/native/java_util_zip_Deflater.cpp;h=c0b2feff196e63a7b85d97cf9ae5bb2583409c28;hb=refs/heads/gingerbread#l53 Code snippet searching for false positives with the zlib checksum: https://gist.github.com/1118177 Signed-off-by: Roberto Tyley <roberto.tyley@guardian.co.uk> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-08-07 22:46:13 +04:00
if ((map[0] & 0x8F) == 0x08 && !(word % 31))
return 0;
else
return 1;
}
unsigned long unpack_object_header_buffer(const unsigned char *buf,
unsigned long len, enum object_type *type, unsigned long *sizep)
{
unsigned shift;
Fix big left-shifts of unsigned char Shifting 'unsigned char' or 'unsigned short' left can result in sign extension errors, since the C integer promotion rules means that the unsigned char/short will get implicitly promoted to a signed 'int' due to the shift (or due to other operations). This normally doesn't matter, but if you shift things up sufficiently, it will now set the sign bit in 'int', and a subsequent cast to a bigger type (eg 'long' or 'unsigned long') will now sign-extend the value despite the original expression being unsigned. One example of this would be something like unsigned long size; unsigned char c; size += c << 24; where despite all the variables being unsigned, 'c << 24' ends up being a signed entity, and will get sign-extended when then doing the addition in an 'unsigned long' type. Since git uses 'unsigned char' pointers extensively, we actually have this bug in a couple of places. I may have missed some, but this is the result of looking at git grep '[^0-9 ][ ]*<<[ ][a-z]' -- '*.c' '*.h' git grep '<<[ ]*24' which catches at least the common byte cases (shifting variables by a variable amount, and shifting by 24 bits). I also grepped for just 'unsigned char' variables in general, and converted the ones that most obviously ended up getting implicitly cast immediately anyway (eg hash_name(), encode_85()). In addition to just avoiding 'unsigned char', this patch also tries to use a common idiom for the delta header size thing. We had three different variations on it: "& 0x7fUL" in one place (getting the sign extension right), and "& ~0x80" and "& 0x7f" in two other places (not getting it right). Apart from making them all just avoid using "unsigned char" at all, I also unified them to then use a simple "& 0x7f". I considered making a sparse extension which warns about doing implicit casts from unsigned types to signed types, but it gets rather complex very quickly, so this is just a hack. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-06-18 04:22:27 +04:00
unsigned long size, c;
unsigned long used = 0;
c = buf[used++];
*type = (c >> 4) & 7;
size = c & 15;
shift = 4;
while (c & 0x80) {
if (len <= used || bitsizeof(long) <= shift) {
error("bad object header");
size = used = 0;
break;
}
c = buf[used++];
size += (c & 0x7f) << shift;
shift += 7;
}
*sizep = size;
return used;
}
int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
{
unsigned long size, used;
static const char valid_loose_object_type[8] = {
0, /* OBJ_EXT */
1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
0, /* "delta" and others are invalid in a loose object */
};
enum object_type type;
/* Get the data stream */
memset(stream, 0, sizeof(*stream));
stream->next_in = map;
stream->avail_in = mapsize;
stream->next_out = buffer;
stream->avail_out = bufsiz;
if (experimental_loose_object(map)) {
/*
* The old experimental format we no longer produce;
* we can still read it.
*/
used = unpack_object_header_buffer(map, mapsize, &type, &size);
if (!used || !valid_loose_object_type[type])
return -1;
map += used;
mapsize -= used;
/* Set up the stream for the rest.. */
stream->next_in = map;
stream->avail_in = mapsize;
git_inflate_init(stream);
/* And generate the fake traditional header */
stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
typename(type), size);
return 0;
}
git_inflate_init(stream);
return git_inflate(stream, 0);
}
2011-06-10 22:52:15 +04:00
static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
{
int bytes = strlen(buffer) + 1;
unsigned char *buf = xmallocz(size);
unsigned long n;
int status = Z_OK;
n = stream->total_out - bytes;
if (n > size)
n = size;
memcpy(buf, (char *) buffer + bytes, n);
bytes = n;
if (bytes <= size) {
/*
* The above condition must be (bytes <= size), not
* (bytes < size). In other words, even though we
* expect no more output and set avail_out to zero,
* the input zlib stream may have bytes that express
* "this concludes the stream", and we *do* want to
* eat that input.
*
* Otherwise we would not be able to test that we
* consumed all the input to reach the expected size;
* we also want to check that zlib tells us that all
* went well with status == Z_STREAM_END at the end.
*/
stream->next_out = buf + bytes;
stream->avail_out = size - bytes;
while (status == Z_OK)
status = git_inflate(stream, Z_FINISH);
}
if (status == Z_STREAM_END && !stream->avail_in) {
git_inflate_end(stream);
return buf;
}
if (status < 0)
error("corrupt loose object '%s'", sha1_to_hex(sha1));
else if (stream->avail_in)
error("garbage at end of loose object '%s'",
sha1_to_hex(sha1));
free(buf);
return NULL;
}
/*
* We used to just use "sscanf()", but that's actually way
* too permissive for what we want to check. So do an anal
* object header parse by hand.
*/
int parse_sha1_header(const char *hdr, unsigned long *sizep)
{
char type[10];
int i;
unsigned long size;
/*
* The type can be at most ten bytes (including the
* terminating '\0' that we add), and is followed by
* a space.
*/
i = 0;
for (;;) {
char c = *hdr++;
if (c == ' ')
break;
type[i++] = c;
if (i >= sizeof(type))
return -1;
}
type[i] = 0;
/*
* The length must follow immediately, and be in canonical
* decimal format (ie "010" is not valid).
*/
size = *hdr++ - '0';
if (size > 9)
return -1;
if (size) {
for (;;) {
unsigned long c = *hdr - '0';
if (c > 9)
break;
hdr++;
size = size * 10 + c;
}
}
*sizep = size;
/*
* The length must be followed by a zero byte
*/
return *hdr ? -1 : type_from_string(type);
}
static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
{
int ret;
2011-06-10 22:52:15 +04:00
git_zstream stream;
char hdr[8192];
ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
return NULL;
return unpack_sha1_rest(&stream, hdr, *size, sha1);
}
unsigned long get_size_from_delta(struct packed_git *p,
struct pack_window **w_curs,
off_t curpos)
{
const unsigned char *data;
unsigned char delta_head[20], *in;
2011-06-10 22:52:15 +04:00
git_zstream stream;
int st;
memset(&stream, 0, sizeof(stream));
stream.next_out = delta_head;
stream.avail_out = sizeof(delta_head);
git_inflate_init(&stream);
do {
in = use_pack(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = git_inflate(&stream, Z_FINISH);
curpos += stream.next_in - in;
} while ((st == Z_OK || st == Z_BUF_ERROR) &&
stream.total_out < sizeof(delta_head));
git_inflate_end(&stream);
if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
error("delta data unpack-initial failed");
return 0;
}
/* Examine the initial part of the delta to figure out
* the result size.
*/
data = delta_head;
/* ignore base size */
get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
/* Read the result size */
return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
}
static off_t get_delta_base(struct packed_git *p,
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window **w_curs,
off_t *curpos,
enum object_type type,
off_t delta_obj_offset)
{
unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
off_t base_offset;
/* use_pack() assured us we have [base_info, base_info + 20)
* as a range that we can look at without walking off the
* end of the mapped window. Its actually the hash size
* that is assured. An OFS_DELTA longer than the hash size
* is stupid, as then a REF_DELTA would be smaller to store.
*/
if (type == OBJ_OFS_DELTA) {
unsigned used = 0;
unsigned char c = base_info[used++];
base_offset = c & 127;
while (c & 128) {
base_offset += 1;
if (!base_offset || MSB(base_offset, 7))
return 0; /* overflow */
c = base_info[used++];
base_offset = (base_offset << 7) + (c & 127);
}
base_offset = delta_obj_offset - base_offset;
if (base_offset <= 0 || base_offset >= delta_obj_offset)
return 0; /* out of bound */
*curpos += used;
} else if (type == OBJ_REF_DELTA) {
/* The base entry _must_ be in the same pack */
base_offset = find_pack_entry_one(base_info, p);
*curpos += 20;
} else
die("I am totally screwed");
return base_offset;
}
/* forward declaration for a mutually recursive function */
static int packed_object_info(struct packed_git *p, off_t offset,
unsigned long *sizep, int *rtype);
static int packed_delta_info(struct packed_git *p,
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window **w_curs,
off_t curpos,
enum object_type type,
off_t obj_offset,
unsigned long *sizep)
{
off_t base_offset;
base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
if (!base_offset)
return OBJ_BAD;
type = packed_object_info(p, base_offset, NULL, NULL);
if (type <= OBJ_NONE) {
struct revindex_entry *revidx;
const unsigned char *base_sha1;
revidx = find_pack_revindex(p, base_offset);
if (!revidx)
return OBJ_BAD;
base_sha1 = nth_packed_object_sha1(p, revidx->nr);
mark_bad_packed_object(p, base_sha1);
type = sha1_object_info(base_sha1, NULL);
if (type <= OBJ_NONE)
return OBJ_BAD;
}
/* We choose to only get the type of the base object and
* ignore potentially corrupt pack file that expects the delta
* based on a base with a wrong size. This saves tons of
* inflate() calls.
*/
if (sizep) {
*sizep = get_size_from_delta(p, w_curs, curpos);
if (*sizep == 0)
type = OBJ_BAD;
}
return type;
}
int unpack_object_header(struct packed_git *p,
struct pack_window **w_curs,
off_t *curpos,
unsigned long *sizep)
{
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
unsigned char *base;
2011-06-10 22:52:15 +04:00
unsigned long left;
unsigned long used;
enum object_type type;
/* use_pack() assures us we have [base, base + 20) available
* as a range that we can look at. (Its actually the hash
* size that is assured.) With our object header encoding
* the maximum deflated object size is 2^137, which is just
* insane, so we know won't exceed what we have been given.
*/
base = use_pack(p, w_curs, *curpos, &left);
used = unpack_object_header_buffer(base, left, &type, sizep);
if (!used) {
type = OBJ_BAD;
} else
*curpos += used;
return type;
}
static int packed_object_info(struct packed_git *p, off_t obj_offset,
unsigned long *sizep, int *rtype)
{
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window *w_curs = NULL;
unsigned long size;
off_t curpos = obj_offset;
enum object_type type;
type = unpack_object_header(p, &w_curs, &curpos, &size);
if (rtype)
*rtype = type; /* representation type */
switch (type) {
case OBJ_OFS_DELTA:
case OBJ_REF_DELTA:
type = packed_delta_info(p, &w_curs, curpos,
type, obj_offset, sizep);
break;
case OBJ_COMMIT:
case OBJ_TREE:
case OBJ_BLOB:
case OBJ_TAG:
if (sizep)
*sizep = size;
break;
default:
error("unknown object type %i at offset %"PRIuMAX" in %s",
type, (uintmax_t)obj_offset, p->pack_name);
type = OBJ_BAD;
}
unuse_pack(&w_curs);
return type;
}
static void *unpack_compressed_entry(struct packed_git *p,
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window **w_curs,
off_t curpos,
unsigned long size)
{
int st;
2011-06-10 22:52:15 +04:00
git_zstream stream;
unsigned char *buffer, *in;
buffer = xmallocz(size);
memset(&stream, 0, sizeof(stream));
stream.next_out = buffer;
Fix incorrect error check while reading deflated pack data The loop in get_size_from_delta() feeds a deflated delta data from the pack stream _until_ we get inflated result of 20 bytes[*] or we reach the end of stream. Side note. This magic number 20 does not have anything to do with the size of the hash we use, but comes from 1a3b55c (reduce delta head inflated size, 2006-10-18). The loop reads like this: do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); curpos += stream.next_in - in; } while ((st == Z_OK || st == Z_BUF_ERROR) && stream.total_out < sizeof(delta_head)); This git_inflate() can return: - Z_STREAM_END, if use_pack() fed it enough input and the delta itself was smaller than 20 bytes; - Z_OK, when some progress has been made; - Z_BUF_ERROR, if no progress is possible, because we either ran out of input (due to corrupt pack), or we ran out of output before we saw the end of the stream. The fix b3118bd (sha1_file: Fix infinite loop when pack is corrupted, 2009-10-14) attempted was against a corruption that appears to be a valid stream that produces a result larger than the output buffer, but we are not even trying to read the stream to the end in this loop. If avail_out becomes zero, total_out will be the same as sizeof(delta_head) so the loop will terminate without the "fix". There is no fix from b3118bd needed for this loop, in other words. The loop in unpack_compressed_entry() is quite a different story. It feeds a deflated stream (either delta or base) and allows the stream to produce output up to what we expect but no more. do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); curpos += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR) This _does_ risk falling into an endless interation, as we can exhaust avail_out if the length we expect is smaller than what the stream wants to produce (due to pack corruption). In such a case, avail_out will become zero and inflate() will return Z_BUF_ERROR, while avail_in may (or may not) be zero. But this is not a right fix: do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); + if (st == Z_BUF_ERROR && (stream.avail_in || !stream.avail_out) + break; /* wants more input??? */ curpos += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR) as Z_BUF_ERROR from inflate() may be telling us that avail_in has also run out before reading the end of stream marker. In such a case, both avail_in and avail_out would be zero, and the loop should iterate to allow the end of stream marker to be seen by inflate from the input stream. The right fix for this loop is likely to be to increment the initial avail_out by one (we allocate one extra byte to terminate it with NUL anyway, so there is no risk to overrun the buffer), and break out if we see that avail_out has become zero, in order to detect that the stream wants to produce more than what we expect. After the loop, we have a check that exactly tests this condition: if ((st != Z_STREAM_END) || stream.total_out != size) { free(buffer); return NULL; } So here is a patch (without my previous botched attempts) to fix this issue. The first hunk reverts the corresponding hunk from b3118bd, and the second hunk is the same fix proposed earlier. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-10-22 10:06:14 +04:00
stream.avail_out = size + 1;
git_inflate_init(&stream);
do {
in = use_pack(p, w_curs, curpos, &stream.avail_in);
stream.next_in = in;
st = git_inflate(&stream, Z_FINISH);
Fix incorrect error check while reading deflated pack data The loop in get_size_from_delta() feeds a deflated delta data from the pack stream _until_ we get inflated result of 20 bytes[*] or we reach the end of stream. Side note. This magic number 20 does not have anything to do with the size of the hash we use, but comes from 1a3b55c (reduce delta head inflated size, 2006-10-18). The loop reads like this: do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); curpos += stream.next_in - in; } while ((st == Z_OK || st == Z_BUF_ERROR) && stream.total_out < sizeof(delta_head)); This git_inflate() can return: - Z_STREAM_END, if use_pack() fed it enough input and the delta itself was smaller than 20 bytes; - Z_OK, when some progress has been made; - Z_BUF_ERROR, if no progress is possible, because we either ran out of input (due to corrupt pack), or we ran out of output before we saw the end of the stream. The fix b3118bd (sha1_file: Fix infinite loop when pack is corrupted, 2009-10-14) attempted was against a corruption that appears to be a valid stream that produces a result larger than the output buffer, but we are not even trying to read the stream to the end in this loop. If avail_out becomes zero, total_out will be the same as sizeof(delta_head) so the loop will terminate without the "fix". There is no fix from b3118bd needed for this loop, in other words. The loop in unpack_compressed_entry() is quite a different story. It feeds a deflated stream (either delta or base) and allows the stream to produce output up to what we expect but no more. do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); curpos += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR) This _does_ risk falling into an endless interation, as we can exhaust avail_out if the length we expect is smaller than what the stream wants to produce (due to pack corruption). In such a case, avail_out will become zero and inflate() will return Z_BUF_ERROR, while avail_in may (or may not) be zero. But this is not a right fix: do { in = use_pack(); stream.next_in = in; st = git_inflate(&stream, Z_FINISH); + if (st == Z_BUF_ERROR && (stream.avail_in || !stream.avail_out) + break; /* wants more input??? */ curpos += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR) as Z_BUF_ERROR from inflate() may be telling us that avail_in has also run out before reading the end of stream marker. In such a case, both avail_in and avail_out would be zero, and the loop should iterate to allow the end of stream marker to be seen by inflate from the input stream. The right fix for this loop is likely to be to increment the initial avail_out by one (we allocate one extra byte to terminate it with NUL anyway, so there is no risk to overrun the buffer), and break out if we see that avail_out has become zero, in order to detect that the stream wants to produce more than what we expect. After the loop, we have a check that exactly tests this condition: if ((st != Z_STREAM_END) || stream.total_out != size) { free(buffer); return NULL; } So here is a patch (without my previous botched attempts) to fix this issue. The first hunk reverts the corresponding hunk from b3118bd, and the second hunk is the same fix proposed earlier. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2009-10-22 10:06:14 +04:00
if (!stream.avail_out)
break; /* the payload is larger than it should be */
curpos += stream.next_in - in;
} while (st == Z_OK || st == Z_BUF_ERROR);
git_inflate_end(&stream);
if ((st != Z_STREAM_END) || stream.total_out != size) {
free(buffer);
return NULL;
}
return buffer;
}
#define MAX_DELTA_CACHE (256)
static size_t delta_base_cached;
static struct delta_base_cache_lru_list {
struct delta_base_cache_lru_list *prev;
struct delta_base_cache_lru_list *next;
} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
static struct delta_base_cache_entry {
struct delta_base_cache_lru_list lru;
void *data;
struct packed_git *p;
off_t base_offset;
unsigned long size;
enum object_type type;
} delta_base_cache[MAX_DELTA_CACHE];
static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
{
unsigned long hash;
hash = (unsigned long)p + (unsigned long)base_offset;
hash += (hash >> 8) + (hash >> 16);
return hash % MAX_DELTA_CACHE;
}
static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
{
unsigned long hash = pack_entry_hash(p, base_offset);
struct delta_base_cache_entry *ent = delta_base_cache + hash;
return (ent->data && ent->p == p && ent->base_offset == base_offset);
}
static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
unsigned long *base_size, enum object_type *type, int keep_cache)
{
void *ret;
unsigned long hash = pack_entry_hash(p, base_offset);
struct delta_base_cache_entry *ent = delta_base_cache + hash;
ret = ent->data;
if (!ret || ent->p != p || ent->base_offset != base_offset)
return unpack_entry(p, base_offset, type, base_size);
if (!keep_cache) {
ent->data = NULL;
ent->lru.next->prev = ent->lru.prev;
ent->lru.prev->next = ent->lru.next;
delta_base_cached -= ent->size;
} else {
ret = xmemdupz(ent->data, ent->size);
}
*type = ent->type;
*base_size = ent->size;
return ret;
}
static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
{
if (ent->data) {
free(ent->data);
ent->data = NULL;
ent->lru.next->prev = ent->lru.prev;
ent->lru.prev->next = ent->lru.next;
delta_base_cached -= ent->size;
}
}
void clear_delta_base_cache(void)
{
unsigned long p;
for (p = 0; p < MAX_DELTA_CACHE; p++)
release_delta_base_cache(&delta_base_cache[p]);
}
static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
void *base, unsigned long base_size, enum object_type type)
{
unsigned long hash = pack_entry_hash(p, base_offset);
struct delta_base_cache_entry *ent = delta_base_cache + hash;
struct delta_base_cache_lru_list *lru;
release_delta_base_cache(ent);
delta_base_cached += base_size;
for (lru = delta_base_cache_lru.next;
delta_base_cached > delta_base_cache_limit
&& lru != &delta_base_cache_lru;
lru = lru->next) {
struct delta_base_cache_entry *f = (void *)lru;
if (f->type == OBJ_BLOB)
release_delta_base_cache(f);
}
for (lru = delta_base_cache_lru.next;
delta_base_cached > delta_base_cache_limit
&& lru != &delta_base_cache_lru;
lru = lru->next) {
struct delta_base_cache_entry *f = (void *)lru;
release_delta_base_cache(f);
}
ent->p = p;
ent->base_offset = base_offset;
ent->type = type;
ent->data = base;
ent->size = base_size;
ent->lru.next = &delta_base_cache_lru;
ent->lru.prev = delta_base_cache_lru.prev;
delta_base_cache_lru.prev->next = &ent->lru;
delta_base_cache_lru.prev = &ent->lru;
}
static void *read_object(const unsigned char *sha1, enum object_type *type,
unsigned long *size);
static void *unpack_delta_entry(struct packed_git *p,
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window **w_curs,
off_t curpos,
unsigned long delta_size,
off_t obj_offset,
enum object_type *type,
unsigned long *sizep)
{
void *delta_data, *result, *base;
unsigned long base_size;
off_t base_offset;
base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
if (!base_offset) {
error("failed to validate delta base reference "
"at offset %"PRIuMAX" from %s",
(uintmax_t)curpos, p->pack_name);
return NULL;
}
unuse_pack(w_curs);
base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
if (!base) {
/*
* We're probably in deep shit, but let's try to fetch
* the required base anyway from another pack or loose.
* This is costly but should happen only in the presence
* of a corrupted pack, and is better than failing outright.
*/
struct revindex_entry *revidx;
const unsigned char *base_sha1;
revidx = find_pack_revindex(p, base_offset);
if (!revidx)
return NULL;
base_sha1 = nth_packed_object_sha1(p, revidx->nr);
error("failed to read delta base object %s"
" at offset %"PRIuMAX" from %s",
sha1_to_hex(base_sha1), (uintmax_t)base_offset,
p->pack_name);
mark_bad_packed_object(p, base_sha1);
base = read_object(base_sha1, type, &base_size);
if (!base)
return NULL;
}
delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
if (!delta_data) {
error("failed to unpack compressed delta "
"at offset %"PRIuMAX" from %s",
(uintmax_t)curpos, p->pack_name);
free(base);
return NULL;
}
result = patch_delta(base, base_size,
delta_data, delta_size,
sizep);
if (!result)
die("failed to apply delta");
free(delta_data);
add_delta_base_cache(p, base_offset, base, base_size, *type);
return result;
}
static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
{
static FILE *log_file;
if (!log_file) {
log_file = fopen(log_pack_access, "w");
if (!log_file) {
error("cannot open pack access log '%s' for writing: %s",
log_pack_access, strerror(errno));
log_pack_access = NULL;
return;
}
}
fprintf(log_file, "%s %"PRIuMAX"\n",
p->pack_name, (uintmax_t)obj_offset);
fflush(log_file);
}
close another possibility for propagating pack corruption Abstract -------- With index v2 we have a per object CRC to allow quick and safe reuse of pack data when repacking. This, however, doesn't currently prevent a stealth corruption from being propagated into a new pack when _not_ reusing pack data as demonstrated by the modification to t5302 included here. The Context ----------- The Git database is all checksummed with SHA1 hashes. Any kind of corruption can be confirmed by verifying this per object hash against corresponding data. However this can be costly to perform systematically and therefore this check is often not performed at run time when accessing the object database. First, the loose object format is entirely compressed with zlib which already provide a CRC verification of its own when inflating data. Any disk corruption would be caught already in this case. Then, packed objects are also compressed with zlib but only for their actual payload. The object headers and delta base references are not deflated for obvious performance reasons, however this leave them vulnerable to potentially undetected disk corruptions. Object types are often validated against the expected type when they're requested, and deflated size must always match the size recorded in the object header, so those cases are pretty much covered as well. Where corruptions could go unnoticed is in the delta base reference. Of course, in the OBJ_REF_DELTA case, the odds for a SHA1 reference to get corrupted so it actually matches the SHA1 of another object with the same size (the delta header stores the expected size of the base object to apply against) are virtually zero. In the OBJ_OFS_DELTA case, the reference is a pack offset which would have to match the start boundary of a different base object but still with the same size, and although this is relatively much more "probable" than in the OBJ_REF_DELTA case, the probability is also about zero in absolute terms. Still, the possibility exists as demonstrated in t5302 and is certainly greater than a SHA1 collision, especially in the OBJ_OFS_DELTA case which is now the default when repacking. Again, repacking by reusing existing pack data is OK since the per object CRC provided by index v2 guards against any such corruptions. What t5302 failed to test is a full repack in such case. The Solution ------------ As unlikely as this kind of stealth corruption can be in practice, it certainly isn't acceptable to propagate it into a freshly created pack. But, because this is so unlikely, we don't want to pay the run time cost associated with extra validation checks all the time either. Furthermore, consequences of such corruption in anything but repacking should be rather visible, and even if it could be quite unpleasant, it still has far less severe consequences than actively creating bad packs. So the best compromize is to check packed object CRC when unpacking objects, and only during the compression/writing phase of a repack, and only when not streaming the result. The cost of this is minimal (less than 1% CPU time), and visible only with a full repack. Someone with a stats background could provide an objective evaluation of this, but I suspect that it's bad RAM that has more potential for data corruptions at this point, even in those cases where this extra check is not performed. Still, it is best to prevent a known hole for corruption when recreating object data into a new pack. What about the streamed pack case? Well, any client receiving a pack must always consider that pack as untrusty and perform full validation anyway, hence no such stealth corruption could be propagated to remote repositoryes already. It is therefore worthless doing local validation in that case. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-31 18:31:08 +03:00
int do_check_packed_object_crc;
void *unpack_entry(struct packed_git *p, off_t obj_offset,
enum object_type *type, unsigned long *sizep)
{
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
struct pack_window *w_curs = NULL;
off_t curpos = obj_offset;
void *data;
if (log_pack_access)
write_pack_access_log(p, obj_offset);
close another possibility for propagating pack corruption Abstract -------- With index v2 we have a per object CRC to allow quick and safe reuse of pack data when repacking. This, however, doesn't currently prevent a stealth corruption from being propagated into a new pack when _not_ reusing pack data as demonstrated by the modification to t5302 included here. The Context ----------- The Git database is all checksummed with SHA1 hashes. Any kind of corruption can be confirmed by verifying this per object hash against corresponding data. However this can be costly to perform systematically and therefore this check is often not performed at run time when accessing the object database. First, the loose object format is entirely compressed with zlib which already provide a CRC verification of its own when inflating data. Any disk corruption would be caught already in this case. Then, packed objects are also compressed with zlib but only for their actual payload. The object headers and delta base references are not deflated for obvious performance reasons, however this leave them vulnerable to potentially undetected disk corruptions. Object types are often validated against the expected type when they're requested, and deflated size must always match the size recorded in the object header, so those cases are pretty much covered as well. Where corruptions could go unnoticed is in the delta base reference. Of course, in the OBJ_REF_DELTA case, the odds for a SHA1 reference to get corrupted so it actually matches the SHA1 of another object with the same size (the delta header stores the expected size of the base object to apply against) are virtually zero. In the OBJ_OFS_DELTA case, the reference is a pack offset which would have to match the start boundary of a different base object but still with the same size, and although this is relatively much more "probable" than in the OBJ_REF_DELTA case, the probability is also about zero in absolute terms. Still, the possibility exists as demonstrated in t5302 and is certainly greater than a SHA1 collision, especially in the OBJ_OFS_DELTA case which is now the default when repacking. Again, repacking by reusing existing pack data is OK since the per object CRC provided by index v2 guards against any such corruptions. What t5302 failed to test is a full repack in such case. The Solution ------------ As unlikely as this kind of stealth corruption can be in practice, it certainly isn't acceptable to propagate it into a freshly created pack. But, because this is so unlikely, we don't want to pay the run time cost associated with extra validation checks all the time either. Furthermore, consequences of such corruption in anything but repacking should be rather visible, and even if it could be quite unpleasant, it still has far less severe consequences than actively creating bad packs. So the best compromize is to check packed object CRC when unpacking objects, and only during the compression/writing phase of a repack, and only when not streaming the result. The cost of this is minimal (less than 1% CPU time), and visible only with a full repack. Someone with a stats background could provide an objective evaluation of this, but I suspect that it's bad RAM that has more potential for data corruptions at this point, even in those cases where this extra check is not performed. Still, it is best to prevent a known hole for corruption when recreating object data into a new pack. What about the streamed pack case? Well, any client receiving a pack must always consider that pack as untrusty and perform full validation anyway, hence no such stealth corruption could be propagated to remote repositoryes already. It is therefore worthless doing local validation in that case. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-31 18:31:08 +03:00
if (do_check_packed_object_crc && p->index_version > 1) {
struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
unsigned long len = revidx[1].offset - obj_offset;
if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
const unsigned char *sha1 =
nth_packed_object_sha1(p, revidx->nr);
error("bad packed object CRC for %s",
sha1_to_hex(sha1));
mark_bad_packed_object(p, sha1);
unuse_pack(&w_curs);
close another possibility for propagating pack corruption Abstract -------- With index v2 we have a per object CRC to allow quick and safe reuse of pack data when repacking. This, however, doesn't currently prevent a stealth corruption from being propagated into a new pack when _not_ reusing pack data as demonstrated by the modification to t5302 included here. The Context ----------- The Git database is all checksummed with SHA1 hashes. Any kind of corruption can be confirmed by verifying this per object hash against corresponding data. However this can be costly to perform systematically and therefore this check is often not performed at run time when accessing the object database. First, the loose object format is entirely compressed with zlib which already provide a CRC verification of its own when inflating data. Any disk corruption would be caught already in this case. Then, packed objects are also compressed with zlib but only for their actual payload. The object headers and delta base references are not deflated for obvious performance reasons, however this leave them vulnerable to potentially undetected disk corruptions. Object types are often validated against the expected type when they're requested, and deflated size must always match the size recorded in the object header, so those cases are pretty much covered as well. Where corruptions could go unnoticed is in the delta base reference. Of course, in the OBJ_REF_DELTA case, the odds for a SHA1 reference to get corrupted so it actually matches the SHA1 of another object with the same size (the delta header stores the expected size of the base object to apply against) are virtually zero. In the OBJ_OFS_DELTA case, the reference is a pack offset which would have to match the start boundary of a different base object but still with the same size, and although this is relatively much more "probable" than in the OBJ_REF_DELTA case, the probability is also about zero in absolute terms. Still, the possibility exists as demonstrated in t5302 and is certainly greater than a SHA1 collision, especially in the OBJ_OFS_DELTA case which is now the default when repacking. Again, repacking by reusing existing pack data is OK since the per object CRC provided by index v2 guards against any such corruptions. What t5302 failed to test is a full repack in such case. The Solution ------------ As unlikely as this kind of stealth corruption can be in practice, it certainly isn't acceptable to propagate it into a freshly created pack. But, because this is so unlikely, we don't want to pay the run time cost associated with extra validation checks all the time either. Furthermore, consequences of such corruption in anything but repacking should be rather visible, and even if it could be quite unpleasant, it still has far less severe consequences than actively creating bad packs. So the best compromize is to check packed object CRC when unpacking objects, and only during the compression/writing phase of a repack, and only when not streaming the result. The cost of this is minimal (less than 1% CPU time), and visible only with a full repack. Someone with a stats background could provide an objective evaluation of this, but I suspect that it's bad RAM that has more potential for data corruptions at this point, even in those cases where this extra check is not performed. Still, it is best to prevent a known hole for corruption when recreating object data into a new pack. What about the streamed pack case? Well, any client receiving a pack must always consider that pack as untrusty and perform full validation anyway, hence no such stealth corruption could be propagated to remote repositoryes already. It is therefore worthless doing local validation in that case. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-31 18:31:08 +03:00
return NULL;
}
}
*type = unpack_object_header(p, &w_curs, &curpos, sizep);
switch (*type) {
case OBJ_OFS_DELTA:
case OBJ_REF_DELTA:
data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
obj_offset, type, sizep);
break;
case OBJ_COMMIT:
case OBJ_TREE:
case OBJ_BLOB:
case OBJ_TAG:
data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
break;
default:
data = NULL;
error("unknown object type %i at offset %"PRIuMAX" in %s",
*type, (uintmax_t)obj_offset, p->pack_name);
}
Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2006-12-23 10:34:08 +03:00
unuse_pack(&w_curs);
return data;
}
const unsigned char *nth_packed_object_sha1(struct packed_git *p,
uint32_t n)
{
const unsigned char *index = p->index_data;
if (!index) {
if (open_pack_index(p))
return NULL;
index = p->index_data;
}
if (n >= p->num_objects)
return NULL;
index += 4 * 256;
if (p->index_version == 1) {
return index + 24 * n + 4;
} else {
index += 8;
return index + 20 * n;
}
}
off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
{
const unsigned char *index = p->index_data;
index += 4 * 256;
if (p->index_version == 1) {
return ntohl(*((uint32_t *)(index + 24 * n)));
} else {
uint32_t off;
index += 8 + p->num_objects * (20 + 4);
off = ntohl(*((uint32_t *)(index + 4 * n)));
if (!(off & 0x80000000))
return off;
index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
ntohl(*((uint32_t *)(index + 4)));
}
}
off_t find_pack_entry_one(const unsigned char *sha1,
struct packed_git *p)
{
const uint32_t *level1_ofs = p->index_data;
const unsigned char *index = p->index_data;
sha1-lookup: more memory efficient search in sorted list of SHA-1 Currently, when looking for a packed object from the pack idx, a simple binary search is used. A conventional binary search loop looks like this: unsigned lo, hi; do { unsigned mi = (lo + hi) / 2; int cmp = "entry pointed at by mi" minus "target"; if (!cmp) return mi; "mi is the wanted one" if (cmp > 0) hi = mi; "mi is larger than target" else lo = mi+1; "mi is smaller than target" } while (lo < hi); "did not find what we wanted" The invariants are: - When entering the loop, 'lo' points at a slot that is never above the target (it could be at the target), 'hi' points at a slot that is guaranteed to be above the target (it can never be at the target). - We find a point 'mi' between 'lo' and 'hi' ('mi' could be the same as 'lo', but never can be as high as 'hi'), and check if 'mi' hits the target. There are three cases: - if it is a hit, we have found what we are looking for; - if it is strictly higher than the target, we set it to 'hi', and repeat the search. - if it is strictly lower than the target, we update 'lo' to one slot after it, because we allow 'lo' to be at the target and 'mi' is known to be below the target. If the loop exits, there is no matching entry. When choosing 'mi', we do not have to take the "middle" but anywhere in between 'lo' and 'hi', as long as lo <= mi < hi is satisfied. When we somehow know that the distance between the target and 'lo' is much shorter than the target and 'hi', we could pick 'mi' that is much closer to 'lo' than (hi+lo)/2, which a conventional binary search would pick. This patch takes advantage of the fact that the SHA-1 is a good hash function, and as long as there are enough entries in the table, we can expect uniform distribution. An entry that begins with for example "deadbeef..." is much likely to appear much later than in the midway of a reasonably populated table. In fact, it can be expected to be near 87% (222/256) from the top of the table. This is a work-in-progress and has switches to allow easier experiments and debugging. Exporting GIT_USE_LOOKUP environment variable enables this code. On my admittedly memory starved machine, with a partial KDE repository (3.0G pack with 95M idx): $ GIT_USE_LOOKUP=t git log -800 --stat HEAD >/dev/null 3.93user 0.16system 0:04.09elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+55588minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -800 --stat HEAD >/dev/null 4.00user 0.15system 0:04.17elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+60258minor)pagefaults 0swaps In the same repository: $ GIT_USE_LOOKUP=t git log -2000 HEAD >/dev/null 0.12user 0.00system 0:00.12elapsed 97%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+4241minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -2000 HEAD >/dev/null 0.05user 0.01system 0:00.07elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+8506minor)pagefaults 0swaps There isn't much time difference, but the number of minor faults seems to show that we are touching much smaller number of pages, which is expected. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-12-29 13:05:47 +03:00
unsigned hi, lo, stride;
static int use_lookup = -1;
static int debug_lookup = -1;
if (debug_lookup < 0)
debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
if (!index) {
if (open_pack_index(p))
return 0;
level1_ofs = p->index_data;
index = p->index_data;
}
if (p->index_version > 1) {
level1_ofs += 2;
index += 8;
}
index += 4 * 256;
hi = ntohl(level1_ofs[*sha1]);
lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
sha1-lookup: more memory efficient search in sorted list of SHA-1 Currently, when looking for a packed object from the pack idx, a simple binary search is used. A conventional binary search loop looks like this: unsigned lo, hi; do { unsigned mi = (lo + hi) / 2; int cmp = "entry pointed at by mi" minus "target"; if (!cmp) return mi; "mi is the wanted one" if (cmp > 0) hi = mi; "mi is larger than target" else lo = mi+1; "mi is smaller than target" } while (lo < hi); "did not find what we wanted" The invariants are: - When entering the loop, 'lo' points at a slot that is never above the target (it could be at the target), 'hi' points at a slot that is guaranteed to be above the target (it can never be at the target). - We find a point 'mi' between 'lo' and 'hi' ('mi' could be the same as 'lo', but never can be as high as 'hi'), and check if 'mi' hits the target. There are three cases: - if it is a hit, we have found what we are looking for; - if it is strictly higher than the target, we set it to 'hi', and repeat the search. - if it is strictly lower than the target, we update 'lo' to one slot after it, because we allow 'lo' to be at the target and 'mi' is known to be below the target. If the loop exits, there is no matching entry. When choosing 'mi', we do not have to take the "middle" but anywhere in between 'lo' and 'hi', as long as lo <= mi < hi is satisfied. When we somehow know that the distance between the target and 'lo' is much shorter than the target and 'hi', we could pick 'mi' that is much closer to 'lo' than (hi+lo)/2, which a conventional binary search would pick. This patch takes advantage of the fact that the SHA-1 is a good hash function, and as long as there are enough entries in the table, we can expect uniform distribution. An entry that begins with for example "deadbeef..." is much likely to appear much later than in the midway of a reasonably populated table. In fact, it can be expected to be near 87% (222/256) from the top of the table. This is a work-in-progress and has switches to allow easier experiments and debugging. Exporting GIT_USE_LOOKUP environment variable enables this code. On my admittedly memory starved machine, with a partial KDE repository (3.0G pack with 95M idx): $ GIT_USE_LOOKUP=t git log -800 --stat HEAD >/dev/null 3.93user 0.16system 0:04.09elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+55588minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -800 --stat HEAD >/dev/null 4.00user 0.15system 0:04.17elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+60258minor)pagefaults 0swaps In the same repository: $ GIT_USE_LOOKUP=t git log -2000 HEAD >/dev/null 0.12user 0.00system 0:00.12elapsed 97%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+4241minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -2000 HEAD >/dev/null 0.05user 0.01system 0:00.07elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+8506minor)pagefaults 0swaps There isn't much time difference, but the number of minor faults seems to show that we are touching much smaller number of pages, which is expected. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-12-29 13:05:47 +03:00
if (p->index_version > 1) {
stride = 20;
} else {
stride = 24;
index += 4;
}
if (debug_lookup)
printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
sha1-lookup: more memory efficient search in sorted list of SHA-1 Currently, when looking for a packed object from the pack idx, a simple binary search is used. A conventional binary search loop looks like this: unsigned lo, hi; do { unsigned mi = (lo + hi) / 2; int cmp = "entry pointed at by mi" minus "target"; if (!cmp) return mi; "mi is the wanted one" if (cmp > 0) hi = mi; "mi is larger than target" else lo = mi+1; "mi is smaller than target" } while (lo < hi); "did not find what we wanted" The invariants are: - When entering the loop, 'lo' points at a slot that is never above the target (it could be at the target), 'hi' points at a slot that is guaranteed to be above the target (it can never be at the target). - We find a point 'mi' between 'lo' and 'hi' ('mi' could be the same as 'lo', but never can be as high as 'hi'), and check if 'mi' hits the target. There are three cases: - if it is a hit, we have found what we are looking for; - if it is strictly higher than the target, we set it to 'hi', and repeat the search. - if it is strictly lower than the target, we update 'lo' to one slot after it, because we allow 'lo' to be at the target and 'mi' is known to be below the target. If the loop exits, there is no matching entry. When choosing 'mi', we do not have to take the "middle" but anywhere in between 'lo' and 'hi', as long as lo <= mi < hi is satisfied. When we somehow know that the distance between the target and 'lo' is much shorter than the target and 'hi', we could pick 'mi' that is much closer to 'lo' than (hi+lo)/2, which a conventional binary search would pick. This patch takes advantage of the fact that the SHA-1 is a good hash function, and as long as there are enough entries in the table, we can expect uniform distribution. An entry that begins with for example "deadbeef..." is much likely to appear much later than in the midway of a reasonably populated table. In fact, it can be expected to be near 87% (222/256) from the top of the table. This is a work-in-progress and has switches to allow easier experiments and debugging. Exporting GIT_USE_LOOKUP environment variable enables this code. On my admittedly memory starved machine, with a partial KDE repository (3.0G pack with 95M idx): $ GIT_USE_LOOKUP=t git log -800 --stat HEAD >/dev/null 3.93user 0.16system 0:04.09elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+55588minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -800 --stat HEAD >/dev/null 4.00user 0.15system 0:04.17elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+60258minor)pagefaults 0swaps In the same repository: $ GIT_USE_LOOKUP=t git log -2000 HEAD >/dev/null 0.12user 0.00system 0:00.12elapsed 97%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+4241minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -2000 HEAD >/dev/null 0.05user 0.01system 0:00.07elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+8506minor)pagefaults 0swaps There isn't much time difference, but the number of minor faults seems to show that we are touching much smaller number of pages, which is expected. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-12-29 13:05:47 +03:00
sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
if (use_lookup < 0)
use_lookup = !!getenv("GIT_USE_LOOKUP");
if (use_lookup) {
int pos = sha1_entry_pos(index, stride, 0,
lo, hi, p->num_objects, sha1);
if (pos < 0)
return 0;
return nth_packed_object_offset(p, pos);
}
do {
unsigned mi = (lo + hi) / 2;
sha1-lookup: more memory efficient search in sorted list of SHA-1 Currently, when looking for a packed object from the pack idx, a simple binary search is used. A conventional binary search loop looks like this: unsigned lo, hi; do { unsigned mi = (lo + hi) / 2; int cmp = "entry pointed at by mi" minus "target"; if (!cmp) return mi; "mi is the wanted one" if (cmp > 0) hi = mi; "mi is larger than target" else lo = mi+1; "mi is smaller than target" } while (lo < hi); "did not find what we wanted" The invariants are: - When entering the loop, 'lo' points at a slot that is never above the target (it could be at the target), 'hi' points at a slot that is guaranteed to be above the target (it can never be at the target). - We find a point 'mi' between 'lo' and 'hi' ('mi' could be the same as 'lo', but never can be as high as 'hi'), and check if 'mi' hits the target. There are three cases: - if it is a hit, we have found what we are looking for; - if it is strictly higher than the target, we set it to 'hi', and repeat the search. - if it is strictly lower than the target, we update 'lo' to one slot after it, because we allow 'lo' to be at the target and 'mi' is known to be below the target. If the loop exits, there is no matching entry. When choosing 'mi', we do not have to take the "middle" but anywhere in between 'lo' and 'hi', as long as lo <= mi < hi is satisfied. When we somehow know that the distance between the target and 'lo' is much shorter than the target and 'hi', we could pick 'mi' that is much closer to 'lo' than (hi+lo)/2, which a conventional binary search would pick. This patch takes advantage of the fact that the SHA-1 is a good hash function, and as long as there are enough entries in the table, we can expect uniform distribution. An entry that begins with for example "deadbeef..." is much likely to appear much later than in the midway of a reasonably populated table. In fact, it can be expected to be near 87% (222/256) from the top of the table. This is a work-in-progress and has switches to allow easier experiments and debugging. Exporting GIT_USE_LOOKUP environment variable enables this code. On my admittedly memory starved machine, with a partial KDE repository (3.0G pack with 95M idx): $ GIT_USE_LOOKUP=t git log -800 --stat HEAD >/dev/null 3.93user 0.16system 0:04.09elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+55588minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -800 --stat HEAD >/dev/null 4.00user 0.15system 0:04.17elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+60258minor)pagefaults 0swaps In the same repository: $ GIT_USE_LOOKUP=t git log -2000 HEAD >/dev/null 0.12user 0.00system 0:00.12elapsed 97%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+4241minor)pagefaults 0swaps Without the patch, the numbers are: $ git log -2000 HEAD >/dev/null 0.05user 0.01system 0:00.07elapsed 100%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (0major+8506minor)pagefaults 0swaps There isn't much time difference, but the number of minor faults seems to show that we are touching much smaller number of pages, which is expected. Signed-off-by: Junio C Hamano <gitster@pobox.com>
2007-12-29 13:05:47 +03:00
int cmp = hashcmp(index + mi * stride, sha1);
if (debug_lookup)
printf("lo %u hi %u rg %u mi %u\n",
lo, hi, hi - lo, mi);
if (!cmp)
return nth_packed_object_offset(p, mi);
if (cmp > 0)
hi = mi;
else
lo = mi+1;
} while (lo < hi);
return 0;
}
pack-objects: protect against disappearing packs It's possible that while pack-objects is running, a simultaneously running prune process might delete a pack that we are interested in. Because we load the pack indices early on, we know that the pack contains our item, but by the time we try to open and map it, it is gone. Since c715f78, we already protect against this in the normal object access code path, but pack-objects accesses the packs at a lower level. In the normal access path, we call find_pack_entry, which will call find_pack_entry_one on each pack index, which does the actual lookup. If it gets a hit, we will actually open and verify the validity of the matching packfile (using c715f78's is_pack_valid). If we can't open it, we'll issue a warning and pretend that we didn't find it, causing us to go on to the next pack (or on to loose objects). Furthermore, we will cache the descriptor to the opened packfile. Which means that later, when we actually try to access the object, we are likely to still have that packfile opened, and won't care if it has been unlinked from the filesystem. Notice the "likely" above. If there is another pack access in the interim, and we run out of descriptors, we could close the pack. And then a later attempt to access the closed pack could fail (we'll try to re-open it, of course, but it may have been deleted). In practice, this doesn't happen because we tend to look up items and then access them immediately. Pack-objects does not follow this code path. Instead, it accesses the packs at a much lower level, using find_pack_entry_one directly. This means we skip the is_pack_valid check, and may end up with the name of a packfile, but no open descriptor. We can add the same is_pack_valid check here. Unfortunately, the access patterns of pack-objects are not quite as nice for keeping lookup and object access together. We look up each object as we find out about it, and the only later when writing the packfile do we necessarily access it. Which means that the opened packfile may be closed in the interim. In practice, however, adding this check still has value, for three reasons. 1. If you have a reasonable number of packs and/or a reasonable file descriptor limit, you can keep all of your packs open simultaneously. If this is the case, then the race is impossible to trigger. 2. Even if you can't keep all packs open at once, you may end up keeping the deleted one open (i.e., you may get lucky). 3. The race window is shortened. You may notice early that the pack is gone, and not try to access it. Triggering the problem without this check means deleting the pack any time after we read the list of index files, but before we access the looked-up objects. Triggering it with this check means deleting the pack means deleting the pack after we do a lookup (and successfully access the packfile), but before we access the object. Which is a smaller window. Acked-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2011-10-14 22:03:48 +04:00
int is_pack_valid(struct packed_git *p)
{
/* An already open pack is known to be valid. */
if (p->pack_fd != -1)
return 1;
/* If the pack has one window completely covering the
* file size, the pack is known to be valid even if
* the descriptor is not currently open.
*/
if (p->windows) {
struct pack_window *w = p->windows;
if (!w->offset && w->len == p->pack_size)
return 1;
}
/* Force the pack to open to prove its valid. */
return !open_packed_git(p);
}
static int fill_pack_entry(const unsigned char *sha1,
struct pack_entry *e,
struct packed_git *p)
{
off_t offset;
if (p->num_bad_objects) {
unsigned i;
for (i = 0; i < p->num_bad_objects; i++)
if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
return 0;
}
offset = find_pack_entry_one(sha1, p);
if (!offset)
return 0;
/*
* We are about to tell the caller where they can locate the
* requested object. We better make sure the packfile is
* still here and can be accessed before supplying that
* answer, as it may have been deleted since the index was
* loaded!
*/
if (!is_pack_valid(p)) {
warning("packfile %s cannot be accessed", p->pack_name);
return 0;
}
e->offset = offset;
e->p = p;
hashcpy(e->sha1, sha1);
return 1;
}
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
{
struct packed_git *p;
prepare_packed_git();
if (!packed_git)
return 0;
if (last_found_pack && fill_pack_entry(sha1, e, last_found_pack))
return 1;
for (p = packed_git; p; p = p->next) {
if (p == last_found_pack || !fill_pack_entry(sha1, e, p))
continue;
last_found_pack = p;
return 1;
}
return 0;
}
struct packed_git *find_sha1_pack(const unsigned char *sha1,
struct packed_git *packs)
{
struct packed_git *p;
for (p = packs; p; p = p->next) {
if (find_pack_entry_one(sha1, p))
return p;
}
return NULL;
}
static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep)
{
int status;
unsigned long mapsize, size;
void *map;
2011-06-10 22:52:15 +04:00
git_zstream stream;
char hdr[32];
map = map_sha1_file(sha1, &mapsize);
if (!map)
return error("unable to find %s", sha1_to_hex(sha1));
if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
status = error("unable to unpack %s header",
sha1_to_hex(sha1));
else if ((status = parse_sha1_header(hdr, &size)) < 0)
status = error("unable to parse %s header", sha1_to_hex(sha1));
else if (sizep)
*sizep = size;
git_inflate_end(&stream);
munmap(map, mapsize);
return status;
}
/* returns enum object_type or negative */
int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
{
struct cached_object *co;
struct pack_entry e;
int status, rtype;
co = find_cached_object(sha1);
if (co) {
if (oi->sizep)
*(oi->sizep) = co->size;
oi->whence = OI_CACHED;
return co->type;
}
if (!find_pack_entry(sha1, &e)) {
/* Most likely it's a loose object. */
status = sha1_loose_object_info(sha1, oi->sizep);
if (status >= 0) {
oi->whence = OI_LOOSE;
return status;
}
/* Not a loose object; someone else may have just packed it. */
reprepare_packed_git();
if (!find_pack_entry(sha1, &e))
return status;
}
status = packed_object_info(e.p, e.offset, oi->sizep, &rtype);
if (status < 0) {
mark_bad_packed_object(e.p, sha1);
status = sha1_object_info_extended(sha1, oi);
} else if (in_delta_base_cache(e.p, e.offset)) {
oi->whence = OI_DBCACHED;
} else {
oi->whence = OI_PACKED;
oi->u.packed.offset = e.offset;
oi->u.packed.pack = e.p;
oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
rtype == OBJ_OFS_DELTA);
}
return status;
}
int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
{
struct object_info oi;
oi.sizep = sizep;
return sha1_object_info_extended(sha1, &oi);
}
static void *read_packed_sha1(const unsigned char *sha1,
enum object_type *type, unsigned long *size)
{
struct pack_entry e;
void *data;
if (!find_pack_entry(sha1, &e))
return NULL;
data = cache_or_unpack_entry(e.p, e.offset, size, type, 1);
if (!data) {
/*
* We're probably in deep shit, but let's try to fetch
* the required object anyway from another pack or loose.
* This should happen only in the presence of a corrupted
* pack, and is better than failing outright.
*/
error("failed to read object %s at offset %"PRIuMAX" from %s",
sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
mark_bad_packed_object(e.p, sha1);
data = read_object(sha1, type, size);
}
return data;
}
int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
unsigned char *sha1)
{
struct cached_object *co;
hash_sha1_file(buf, len, typename(type), sha1);
if (has_sha1_file(sha1) || find_cached_object(sha1))
return 0;
if (cached_object_alloc <= cached_object_nr) {
cached_object_alloc = alloc_nr(cached_object_alloc);
cached_objects = xrealloc(cached_objects,
sizeof(*cached_objects) *
cached_object_alloc);
}
co = &cached_objects[cached_object_nr++];
co->size = len;
co->type = type;
co->buf = xmalloc(len);
memcpy(co->buf, buf, len);
hashcpy(co->sha1, sha1);
return 0;
}
static void *read_object(const unsigned char *sha1, enum object_type *type,
unsigned long *size)
{
unsigned long mapsize;
void *map, *buf;
struct cached_object *co;
co = find_cached_object(sha1);
if (co) {
*type = co->type;
*size = co->size;
return xmemdupz(co->buf, co->size);
}
buf = read_packed_sha1(sha1, type, size);
if (buf)
return buf;
map = map_sha1_file(sha1, &mapsize);
if (map) {
buf = unpack_sha1_file(map, mapsize, type, size, sha1);
munmap(map, mapsize);
return buf;
}
reprepare_packed_git();
return read_packed_sha1(sha1, type, size);
}
/*
* This function dies on corrupt objects; the callers who want to
* deal with them should arrange to call read_object() and give error
* messages themselves.
*/
void *read_sha1_file_extended(const unsigned char *sha1,
enum object_type *type,
unsigned long *size,
unsigned flag)
{
void *data;
char *path;
const struct packed_git *p;
const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
? lookup_replace_object(sha1) : sha1;
errno = 0;
data = read_object(repl, type, size);
if (data)
return data;
if (errno && errno != ENOENT)
die_errno("failed to read object %s", sha1_to_hex(sha1));
/* die if we replaced an object with one that does not exist */
if (repl != sha1)
die("replacement %s not found for %s",
sha1_to_hex(repl), sha1_to_hex(sha1));
if (has_loose_object(repl)) {
path = sha1_file_name(sha1);
die("loose object %s (stored in %s) is corrupt",
sha1_to_hex(repl), path);
}
if ((p = has_packed_and_bad(repl)) != NULL)
die("packed object %s (stored in %s) is corrupt",
sha1_to_hex(repl), p->pack_name);
return NULL;
}
void *read_object_with_reference(const unsigned char *sha1,
const char *required_type_name,
unsigned long *size,
unsigned char *actual_sha1_return)
{
enum object_type type, required_type;
void *buffer;
unsigned long isize;
unsigned char actual_sha1[20];
required_type = type_from_string(required_type_name);
hashcpy(actual_sha1, sha1);
while (1) {
int ref_length = -1;
const char *ref_type = NULL;
buffer = read_sha1_file(actual_sha1, &type, &isize);
if (!buffer)
return NULL;
if (type == required_type) {
*size = isize;
if (actual_sha1_return)
hashcpy(actual_sha1_return, actual_sha1);
return buffer;
}
/* Handle references */
else if (type == OBJ_COMMIT)
ref_type = "tree ";
else if (type == OBJ_TAG)
ref_type = "object ";
else {
free(buffer);
return NULL;
}
ref_length = strlen(ref_type);
if (ref_length + 40 > isize ||
memcmp(buffer, ref_type, ref_length) ||
get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
free(buffer);
return NULL;
}
free(buffer);
/* Now we have the ID of the referred-to object in
* actual_sha1. Check again. */
}
}
static void write_sha1_file_prepare(const void *buf, unsigned long len,
const char *type, unsigned char *sha1,
char *hdr, int *hdrlen)
{
fix openssl headers conflicting with custom SHA1 implementations On ARM I have the following compilation errors: CC fast-import.o In file included from cache.h:8, from builtin.h:6, from fast-import.c:142: arm/sha1.h:14: error: conflicting types for 'SHA_CTX' /usr/include/openssl/sha.h:105: error: previous declaration of 'SHA_CTX' was here arm/sha1.h:16: error: conflicting types for 'SHA1_Init' /usr/include/openssl/sha.h:115: error: previous declaration of 'SHA1_Init' was here arm/sha1.h:17: error: conflicting types for 'SHA1_Update' /usr/include/openssl/sha.h:116: error: previous declaration of 'SHA1_Update' was here arm/sha1.h:18: error: conflicting types for 'SHA1_Final' /usr/include/openssl/sha.h:117: error: previous declaration of 'SHA1_Final' was here make: *** [fast-import.o] Error 1 This is because openssl header files are always included in git-compat-util.h since commit 684ec6c63c whenever NO_OPENSSL is not set, which somehow brings in <openssl/sha1.h> clashing with the custom ARM version. Compilation of git is probably broken on PPC too for the same reason. Turns out that the only file requiring openssl/ssl.h and openssl/err.h is imap-send.c. But only moving those problematic includes there doesn't solve the issue as it also includes cache.h which brings in the conflicting local SHA1 header file. As suggested by Jeff King, the best solution is to rename our references to SHA1 functions and structure to something git specific, and define those according to the implementation used. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
2008-10-01 22:05:20 +04:00
git_SHA_CTX c;
/* Generate the header */
*hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
/* Sha1.. */
fix openssl headers conflicting with custom SHA1 implementations On ARM I have the following compilation errors: CC fast-import.o In file included from cache.h:8, from builtin.h:6, from fast-import.c:142: arm/sha1.h:14: error: conflicting types for 'SHA_CTX' /usr/include/openssl/sha.h:105: error: previous declaration of 'SHA_CTX' was here arm/sha1.h:16: error: conflicting types for 'SHA1_Init' /usr/include/openssl/sha.h:115: error: previous declaration of 'SHA1_Init' was here arm/sha1.h:17: error: conflicting types for 'SHA1_Update' /usr/include/openssl/sha.h:116: error: previous declaration of 'SHA1_Update' was here arm/sha1.h:18: error: conflicting types for 'SHA1_Final' /usr/include/openssl/sha.h:117: error: previous declaration of 'SHA1_Final' was here make: *** [fast-import.o] Error 1 This is because openssl header files are always included in git-compat-util.h since commit 684ec6c63c whenever NO_OPENSSL is not set, which somehow brings in <openssl/sha1.h> clashing with the custom ARM version. Compilation of git is probably broken on PPC too for the same reason. Turns out that the only file requiring openssl/ssl.h and openssl/err.h is imap-send.c. But only moving those problematic includes there doesn't solve the issue as it also includes cache.h which brings in the conflicting local SHA1 header file. As suggested by Jeff King, the best solution is to rename our references to SHA1 functions and structure to something git specific, and define those according to the implementation used. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
2008-10-01 22:05:20 +04:00
git_SHA1_Init(&c);
git_SHA1_Update(&c, hdr, *hdrlen);
git_SHA1_Update(&c, buf, len);
git_SHA1_Final(sha1, &c);
}
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
/*
* Move the just written object into its final resting place.
* NEEDSWORK: this should be renamed to finalize_temp_file() as
* "moving" is only a part of what it does, when no patch between
* master to pu changes the call sites of this function.
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
*/
int move_temp_to_file(const char *tmpfile, const char *filename)
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
{
int ret = 0;
if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
goto try_rename;
else if (link(tmpfile, filename))
ret = errno;
/*
* Coda hack - coda doesn't like cross-directory links,
* so we fall back to a rename, which will mean that it
* won't be able to check collisions, but that's not a
* big deal.
*
* The same holds for FAT formatted media.
*
* When this succeeds, we just return. We have nothing
* left to unlink.
*/
if (ret && ret != EEXIST) {
try_rename:
if (!rename(tmpfile, filename))
goto out;
ret = errno;
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
}
unlink_or_warn(tmpfile);
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
if (ret) {
if (ret != EEXIST) {
return error("unable to write sha1 filename %s: %s", filename, strerror(ret));
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
}
/* FIXME!!! Collision check here ? */
}
out:
if (adjust_shared_perm(filename))
return error("unable to set permission to '%s'", filename);
Create object subdirectories on demand This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2005-10-09 02:54:01 +04:00
return 0;
}
static int write_buffer(int fd, const void *buf, size_t len)
{
if (write_in_full(fd, buf, len) < 0)
return error("file write error (%s)", strerror(errno));
return 0;
}
int hash_sha1_file(const void *buf, unsigned long len, const char *type,
unsigned char *sha1)
{
char hdr[32];
int hdrlen;
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
return 0;
}
/* Finalize a file on disk, and close it. */
static void close_sha1_file(int fd)
{
if (fsync_object_files)
fsync_or_die(fd, "sha1 file");
if (close(fd) != 0)
die_errno("error when closing sha1 file");
}
/* Size of directory component, including the ending '/' */
static inline int directory_size(const char *filename)
{
const char *s = strrchr(filename, '/');
if (!s)
return 0;
return s - filename + 1;
}
/*
* This creates a temporary file in the same directory as the final
* 'filename'
*
* We want to avoid cross-directory filename renames, because those
* can have problems on various filesystems (FAT, NFS, Coda).
*/
static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename)
{
int fd, dirlen = directory_size(filename);
if (dirlen + 20 > bufsiz) {
errno = ENAMETOOLONG;
return -1;
}
memcpy(buffer, filename, dirlen);
strcpy(buffer + dirlen, "tmp_obj_XXXXXX");
fd = git_mkstemp_mode(buffer, 0444);
if (fd < 0 && dirlen && errno == ENOENT) {
/* Make sure the directory exists */
memcpy(buffer, filename, dirlen);
buffer[dirlen-1] = 0;
if (mkdir(buffer, 0777) || adjust_shared_perm(buffer))
return -1;
/* Try again */
strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX");
fd = git_mkstemp_mode(buffer, 0444);
}
return fd;
}
static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
const void *buf, unsigned long len, time_t mtime)
{
int fd, ret;
unsigned char compressed[4096];
2011-06-10 22:52:15 +04:00
git_zstream stream;
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
git_SHA_CTX c;
unsigned char parano_sha1[20];
char *filename;
static char tmp_file[PATH_MAX];
filename = sha1_file_name(sha1);
fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename);
if (fd < 0) {
if (errno == EACCES)
return error("insufficient permission for adding an object to repository database %s", get_object_directory());
else
return error("unable to create temporary file: %s", strerror(errno));
}
/* Set it up */
memset(&stream, 0, sizeof(stream));
git_deflate_init(&stream, zlib_compression_level);
stream.next_out = compressed;
stream.avail_out = sizeof(compressed);
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
git_SHA1_Init(&c);
/* First header.. */
stream.next_in = (unsigned char *)hdr;
stream.avail_in = hdrlen;
while (git_deflate(&stream, 0) == Z_OK)
; /* nothing */
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
git_SHA1_Update(&c, hdr, hdrlen);
/* Then the data itself.. */
stream.next_in = (void *)buf;
stream.avail_in = len;
do {
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
unsigned char *in0 = stream.next_in;
ret = git_deflate(&stream, Z_FINISH);
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
git_SHA1_Update(&c, in0, stream.next_in - in0);
if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
die("unable to write sha1 file");
stream.next_out = compressed;
stream.avail_out = sizeof(compressed);
} while (ret == Z_OK);
if (ret != Z_STREAM_END)
die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
ret = git_deflate_end_gently(&stream);
if (ret != Z_OK)
die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
sha1_file: be paranoid when creating loose objects We don't want the data being deflated and stored into loose objects to be different from what we expect. While the deflated data is protected by a CRC which is good enough for safe data retrieval operations, we still want to be doubly sure that the source data used at object creation time is still what we expected once that data has been deflated and its CRC32 computed. The most plausible data corruption may occur if the source file is modified while Git is deflating and writing it out in a loose object. Or Git itself could have a bug causing memory corruption. Or even bad RAM could cause trouble. So it is best to make sure everything is coherent and checksum protected from beginning to end. To do so we compute the SHA1 of the data being deflated _after_ the deflate operation has consumed that data, and make sure it matches with the expected SHA1. This way we can rely on the CRC32 checked by the inflate operation to provide a good indication that the data is still coherent with its SHA1 hash. One pathological case we ignore is when the data is modified before (or during) deflate call, but changed back before it is hashed. There is some overhead of course. Using 'git add' on a set of large files: Before: real 0m25.210s user 0m23.783s sys 0m1.408s After: real 0m26.537s user 0m25.175s sys 0m1.358s The overhead is around 5% for full data coherency guarantee. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2010-02-21 23:48:06 +03:00
git_SHA1_Final(parano_sha1, &c);
if (hashcmp(sha1, parano_sha1) != 0)
die("confused by unstable object source data for %s", sha1_to_hex(sha1));
close_sha1_file(fd);
if (mtime) {
struct utimbuf utb;
utb.actime = mtime;
utb.modtime = mtime;
if (utime(tmp_file, &utb) < 0)
warning("failed utime() on %s: %s",
tmp_file, strerror(errno));
}
return move_temp_to_file(tmp_file, filename);
}
int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
{
unsigned char sha1[20];
char hdr[32];
int hdrlen;
/* Normally if we have it in the pack then we do not bother writing
* it out into .git/objects/??/?{38} file.
*/
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
if (returnsha1)
hashcpy(returnsha1, sha1);
if (has_sha1_file(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
}
int force_object_loose(const unsigned char *sha1, time_t mtime)
{
void *buf;
unsigned long len;
enum object_type type;
char hdr[32];
int hdrlen;
int ret;
if (has_loose_object(sha1))
return 0;
buf = read_packed_sha1(sha1, &type, &len);
if (!buf)
return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
free(buf);
return ret;
}
int has_pack_index(const unsigned char *sha1)
{
struct stat st;
if (stat(sha1_pack_index_name(sha1), &st))
return 0;
return 1;
}
int has_sha1_pack(const unsigned char *sha1)
{
struct pack_entry e;
return find_pack_entry(sha1, &e);
}
int has_sha1_file(const unsigned char *sha1)
{
struct pack_entry e;
if (find_pack_entry(sha1, &e))
return 1;
return has_loose_object(sha1);
}
static void check_tree(const void *buf, size_t size)
{
struct tree_desc desc;
struct name_entry entry;
init_tree_desc(&desc, buf, size);
while (tree_entry(&desc, &entry))
/* do nothing
* tree_entry() will die() on malformed entries */
;
}
static void check_commit(const void *buf, size_t size)
{
struct commit c;
memset(&c, 0, sizeof(c));
if (parse_commit_buffer(&c, buf, size))
die("corrupt commit");
}
static void check_tag(const void *buf, size_t size)
{
struct tag t;
memset(&t, 0, sizeof(t));
if (parse_tag_buffer(&t, buf, size))
die("corrupt tag");
}
static int index_mem(unsigned char *sha1, void *buf, size_t size,
enum object_type type,
const char *path, unsigned flags)
{
Lazy man's auto-CRLF It currently does NOT know about file attributes, so it does its conversion purely based on content. Maybe that is more in the "git philosophy" anyway, since content is king, but I think we should try to do the file attributes to turn it off on demand. Anyway, BY DEFAULT it is off regardless, because it requires a [core] AutoCRLF = true in your config file to be enabled. We could make that the default for Windows, of course, the same way we do some other things (filemode etc). But you can actually enable it on UNIX, and it will cause: - "git update-index" will write blobs without CRLF - "git diff" will diff working tree files without CRLF - "git checkout" will write files to the working tree _with_ CRLF and things work fine. Funnily, it actually shows an odd file in git itself: git clone -n git test-crlf cd test-crlf git config core.autocrlf true git checkout git diff shows a diff for "Documentation/docbook-xsl.css". Why? Because we have actually checked in that file *with* CRLF! So when "core.autocrlf" is true, we'll always generate a *different* hash for it in the index, because the index hash will be for the content _without_ CRLF. Is this complete? I dunno. It seems to work for me. It doesn't use the filename at all right now, and that's probably a deficiency (we could certainly make the "is_binary()" heuristics also take standard filename heuristics into account). I don't pass in the filename at all for the "index_fd()" case (git-update-index), so that would need to be passed around, but this actually works fine. NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours truly. I will not guarantee that they work at all reasonable. Caveat emptor. But it _is_ simple, and it _is_ safe, since it's all off by default. The patch is pretty simple - the biggest part is the new "convert.c" file, but even that is really just basic stuff that anybody can write in "Teaching C 101" as a final project for their first class in programming. Not to say that it's bug-free, of course - but at least we're not talking about rocket surgery here. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 22:07:23 +03:00
int ret, re_allocated = 0;
int write_object = flags & HASH_WRITE_OBJECT;
if (!type)
type = OBJ_BLOB;
Lazy man's auto-CRLF It currently does NOT know about file attributes, so it does its conversion purely based on content. Maybe that is more in the "git philosophy" anyway, since content is king, but I think we should try to do the file attributes to turn it off on demand. Anyway, BY DEFAULT it is off regardless, because it requires a [core] AutoCRLF = true in your config file to be enabled. We could make that the default for Windows, of course, the same way we do some other things (filemode etc). But you can actually enable it on UNIX, and it will cause: - "git update-index" will write blobs without CRLF - "git diff" will diff working tree files without CRLF - "git checkout" will write files to the working tree _with_ CRLF and things work fine. Funnily, it actually shows an odd file in git itself: git clone -n git test-crlf cd test-crlf git config core.autocrlf true git checkout git diff shows a diff for "Documentation/docbook-xsl.css". Why? Because we have actually checked in that file *with* CRLF! So when "core.autocrlf" is true, we'll always generate a *different* hash for it in the index, because the index hash will be for the content _without_ CRLF. Is this complete? I dunno. It seems to work for me. It doesn't use the filename at all right now, and that's probably a deficiency (we could certainly make the "is_binary()" heuristics also take standard filename heuristics into account). I don't pass in the filename at all for the "index_fd()" case (git-update-index), so that would need to be passed around, but this actually works fine. NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours truly. I will not guarantee that they work at all reasonable. Caveat emptor. But it _is_ simple, and it _is_ safe, since it's all off by default. The patch is pretty simple - the biggest part is the new "convert.c" file, but even that is really just basic stuff that anybody can write in "Teaching C 101" as a final project for their first class in programming. Not to say that it's bug-free, of course - but at least we're not talking about rocket surgery here. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 22:07:23 +03:00
/*
* Convert blobs to git internal format
*/
if ((type == OBJ_BLOB) && path) {
struct strbuf nbuf = STRBUF_INIT;
safecrlf: Add mechanism to warn about irreversible crlf conversions CRLF conversion bears a slight chance of corrupting data. autocrlf=true will convert CRLF to LF during commit and LF to CRLF during checkout. A file that contains a mixture of LF and CRLF before the commit cannot be recreated by git. For text files this is the right thing to do: it corrects line endings such that we have only LF line endings in the repository. But for binary files that are accidentally classified as text the conversion can corrupt data. If you recognize such corruption early you can easily fix it by setting the conversion type explicitly in .gitattributes. Right after committing you still have the original file in your work tree and this file is not yet corrupted. You can explicitly tell git that this file is binary and git will handle the file appropriately. Unfortunately, the desired effect of cleaning up text files with mixed line endings and the undesired effect of corrupting binary files cannot be distinguished. In both cases CRLFs are removed in an irreversible way. For text files this is the right thing to do because CRLFs are line endings, while for binary files converting CRLFs corrupts data. This patch adds a mechanism that can either warn the user about an irreversible conversion or can even refuse to convert. The mechanism is controlled by the variable core.safecrlf, with the following values: - false: disable safecrlf mechanism - warn: warn about irreversible conversions - true: refuse irreversible conversions The default is to warn. Users are only affected by this default if core.autocrlf is set. But the current default of git is to leave core.autocrlf unset, so users will not see warnings unless they deliberately chose to activate the autocrlf mechanism. The safecrlf mechanism's details depend on the git command. The general principles when safecrlf is active (not false) are: - we warn/error out if files in the work tree can modified in an irreversible way without giving the user a chance to backup the original file. - for read-only operations that do not modify files in the work tree we do not not print annoying warnings. There are exceptions. Even though... - "git add" itself does not touch the files in the work tree, the next checkout would, so the safety triggers; - "git apply" to update a text file with a patch does touch the files in the work tree, but the operation is about text files and CRLF conversion is about fixing the line ending inconsistencies, so the safety does not trigger; - "git diff" itself does not touch the files in the work tree, it is often run to inspect the changes you intend to next "git add". To catch potential problems early, safety triggers. The concept of a safety check was originally proposed in a similar way by Linus Torvalds. Thanks to Dimitry Potapov for insisting on getting the naked LF/autocrlf=true case right. Signed-off-by: Steffen Prohaska <prohaska@zib.de>
2008-02-06 14:25:58 +03:00
if (convert_to_git(path, buf, size, &nbuf,
write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
buf = strbuf_detach(&nbuf, &size);
Lazy man's auto-CRLF It currently does NOT know about file attributes, so it does its conversion purely based on content. Maybe that is more in the "git philosophy" anyway, since content is king, but I think we should try to do the file attributes to turn it off on demand. Anyway, BY DEFAULT it is off regardless, because it requires a [core] AutoCRLF = true in your config file to be enabled. We could make that the default for Windows, of course, the same way we do some other things (filemode etc). But you can actually enable it on UNIX, and it will cause: - "git update-index" will write blobs without CRLF - "git diff" will diff working tree files without CRLF - "git checkout" will write files to the working tree _with_ CRLF and things work fine. Funnily, it actually shows an odd file in git itself: git clone -n git test-crlf cd test-crlf git config core.autocrlf true git checkout git diff shows a diff for "Documentation/docbook-xsl.css". Why? Because we have actually checked in that file *with* CRLF! So when "core.autocrlf" is true, we'll always generate a *different* hash for it in the index, because the index hash will be for the content _without_ CRLF. Is this complete? I dunno. It seems to work for me. It doesn't use the filename at all right now, and that's probably a deficiency (we could certainly make the "is_binary()" heuristics also take standard filename heuristics into account). I don't pass in the filename at all for the "index_fd()" case (git-update-index), so that would need to be passed around, but this actually works fine. NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours truly. I will not guarantee that they work at all reasonable. Caveat emptor. But it _is_ simple, and it _is_ safe, since it's all off by default. The patch is pretty simple - the biggest part is the new "convert.c" file, but even that is really just basic stuff that anybody can write in "Teaching C 101" as a final project for their first class in programming. Not to say that it's bug-free, of course - but at least we're not talking about rocket surgery here. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 22:07:23 +03:00
re_allocated = 1;
}
}
if (flags & HASH_FORMAT_CHECK) {
if (type == OBJ_TREE)
check_tree(buf, size);
if (type == OBJ_COMMIT)
check_commit(buf, size);
if (type == OBJ_TAG)
check_tag(buf, size);
}
Lazy man's auto-CRLF It currently does NOT know about file attributes, so it does its conversion purely based on content. Maybe that is more in the "git philosophy" anyway, since content is king, but I think we should try to do the file attributes to turn it off on demand. Anyway, BY DEFAULT it is off regardless, because it requires a [core] AutoCRLF = true in your config file to be enabled. We could make that the default for Windows, of course, the same way we do some other things (filemode etc). But you can actually enable it on UNIX, and it will cause: - "git update-index" will write blobs without CRLF - "git diff" will diff working tree files without CRLF - "git checkout" will write files to the working tree _with_ CRLF and things work fine. Funnily, it actually shows an odd file in git itself: git clone -n git test-crlf cd test-crlf git config core.autocrlf true git checkout git diff shows a diff for "Documentation/docbook-xsl.css". Why? Because we have actually checked in that file *with* CRLF! So when "core.autocrlf" is true, we'll always generate a *different* hash for it in the index, because the index hash will be for the content _without_ CRLF. Is this complete? I dunno. It seems to work for me. It doesn't use the filename at all right now, and that's probably a deficiency (we could certainly make the "is_binary()" heuristics also take standard filename heuristics into account). I don't pass in the filename at all for the "index_fd()" case (git-update-index), so that would need to be passed around, but this actually works fine. NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours truly. I will not guarantee that they work at all reasonable. Caveat emptor. But it _is_ simple, and it _is_ safe, since it's all off by default. The patch is pretty simple - the biggest part is the new "convert.c" file, but even that is really just basic stuff that anybody can write in "Teaching C 101" as a final project for their first class in programming. Not to say that it's bug-free, of course - but at least we're not talking about rocket surgery here. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 22:07:23 +03:00
if (write_object)
ret = write_sha1_file(buf, size, typename(type), sha1);
else
ret = hash_sha1_file(buf, size, typename(type), sha1);
if (re_allocated)
Lazy man's auto-CRLF It currently does NOT know about file attributes, so it does its conversion purely based on content. Maybe that is more in the "git philosophy" anyway, since content is king, but I think we should try to do the file attributes to turn it off on demand. Anyway, BY DEFAULT it is off regardless, because it requires a [core] AutoCRLF = true in your config file to be enabled. We could make that the default for Windows, of course, the same way we do some other things (filemode etc). But you can actually enable it on UNIX, and it will cause: - "git update-index" will write blobs without CRLF - "git diff" will diff working tree files without CRLF - "git checkout" will write files to the working tree _with_ CRLF and things work fine. Funnily, it actually shows an odd file in git itself: git clone -n git test-crlf cd test-crlf git config core.autocrlf true git checkout git diff shows a diff for "Documentation/docbook-xsl.css". Why? Because we have actually checked in that file *with* CRLF! So when "core.autocrlf" is true, we'll always generate a *different* hash for it in the index, because the index hash will be for the content _without_ CRLF. Is this complete? I dunno. It seems to work for me. It doesn't use the filename at all right now, and that's probably a deficiency (we could certainly make the "is_binary()" heuristics also take standard filename heuristics into account). I don't pass in the filename at all for the "index_fd()" case (git-update-index), so that would need to be passed around, but this actually works fine. NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours truly. I will not guarantee that they work at all reasonable. Caveat emptor. But it _is_ simple, and it _is_ safe, since it's all off by default. The patch is pretty simple - the biggest part is the new "convert.c" file, but even that is really just basic stuff that anybody can write in "Teaching C 101" as a final project for their first class in programming. Not to say that it's bug-free, of course - but at least we're not talking about rocket surgery here. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-02-13 22:07:23 +03:00
free(buf);
return ret;
}
static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
const char *path, unsigned flags)
{
struct strbuf sbuf = STRBUF_INIT;
int ret;
if (strbuf_read(&sbuf, fd, 4096) >= 0)
ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
else
ret = -1;
strbuf_release(&sbuf);
return ret;
}
#define SMALL_FILE_SIZE (32*1024)
static int index_core(unsigned char *sha1, int fd, size_t size,
enum object_type type, const char *path,
unsigned flags)
{
int ret;
if (!size) {
ret = index_mem(sha1, NULL, size, type, path, flags);
} else if (size <= SMALL_FILE_SIZE) {
char *buf = xmalloc(size);
if (size == read_in_full(fd, buf, size))
ret = index_mem(sha1, buf, size, type, path, flags);
else
ret = error("short read %s", strerror(errno));
free(buf);
} else {
void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
ret = index_mem(sha1, buf, size, type, path, flags);
munmap(buf, size);
}
return ret;
}
/*
* This creates one packfile per large blob unless bulk-checkin
* machinery is "plugged".
*
* This also bypasses the usual "convert-to-git" dance, and that is on
* purpose. We could write a streaming version of the converting
* functions and insert that before feeding the data to fast-import
do not stream large files to pack when filters are in use Because git's object format requires us to specify the number of bytes in the object in its header, we must know the size before streaming a blob into the object database. This is not a problem when adding a regular file, as we can get the size from stat(). However, when filters are in use (such as autocrlf, or the ident, filter, or eol gitattributes), we have no idea what the ultimate size will be. The current code just punts on the whole issue and ignores filter configuration entirely for files larger than core.bigfilethreshold. This can generate confusing results if you use filters for large binary files, as the filter will suddenly stop working as the file goes over a certain size. Rather than try to handle unknown input sizes with streaming, this patch just turns off the streaming optimization when filters are in use. This has a slight performance regression in a very specific case: if you have autocrlf on, but no gitattributes, a large binary file will avoid the streaming code path because we don't know beforehand whether it will need conversion or not. But if you are handling large binary files, you should be marking them as such via attributes (or at least not using autocrlf, and instead marking your text files as such). And the flip side is that if you have a large _non_-binary file, there is a correctness improvement; before we did not apply the conversion at all. The first half of the new t1051 script covers these failures on input. The second half tests the matching output code paths. These already work correctly, and do not need any adjustment. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-25 02:10:17 +04:00
* (or equivalent in-core API described above). However, that is
* somewhat complicated, as we do not know the size of the filter
* result, which we need to know beforehand when writing a git object.
* Since the primary motivation for trying to stream from the working
* tree file and to avoid mmaping it in core is to deal with large
* binary blobs, they generally do not want to get any conversion, and
* callers should avoid this code path when filters are requested.
*/
static int index_stream(unsigned char *sha1, int fd, size_t size,
enum object_type type, const char *path,
unsigned flags)
{
return index_bulk_checkin(sha1, fd, size, type, path, flags);
}
int index_fd(unsigned char *sha1, int fd, struct stat *st,
enum object_type type, const char *path, unsigned flags)
{
int ret;
size_t size = xsize_t(st->st_size);
if (!S_ISREG(st->st_mode))
ret = index_pipe(sha1, fd, type, path, flags);
do not stream large files to pack when filters are in use Because git's object format requires us to specify the number of bytes in the object in its header, we must know the size before streaming a blob into the object database. This is not a problem when adding a regular file, as we can get the size from stat(). However, when filters are in use (such as autocrlf, or the ident, filter, or eol gitattributes), we have no idea what the ultimate size will be. The current code just punts on the whole issue and ignores filter configuration entirely for files larger than core.bigfilethreshold. This can generate confusing results if you use filters for large binary files, as the filter will suddenly stop working as the file goes over a certain size. Rather than try to handle unknown input sizes with streaming, this patch just turns off the streaming optimization when filters are in use. This has a slight performance regression in a very specific case: if you have autocrlf on, but no gitattributes, a large binary file will avoid the streaming code path because we don't know beforehand whether it will need conversion or not. But if you are handling large binary files, you should be marking them as such via attributes (or at least not using autocrlf, and instead marking your text files as such). And the flip side is that if you have a large _non_-binary file, there is a correctness improvement; before we did not apply the conversion at all. The first half of the new t1051 script covers these failures on input. The second half tests the matching output code paths. These already work correctly, and do not need any adjustment. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-25 02:10:17 +04:00
else if (size <= big_file_threshold || type != OBJ_BLOB ||
(path && would_convert_to_git(path, NULL, 0, 0)))
ret = index_core(sha1, fd, size, type, path, flags);
else
ret = index_stream(sha1, fd, size, type, path, flags);
close(fd);
return ret;
}
int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
{
int fd;
struct strbuf sb = STRBUF_INIT;
switch (st->st_mode & S_IFMT) {
case S_IFREG:
fd = open(path, O_RDONLY);
if (fd < 0)
return error("open(\"%s\"): %s", path,
strerror(errno));
if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
return error("%s: failed to insert into database",
path);
break;
case S_IFLNK:
if (strbuf_readlink(&sb, path, st->st_size)) {
char *errstr = strerror(errno);
return error("readlink(\"%s\"): %s", path,
errstr);
}
if (!(flags & HASH_WRITE_OBJECT))
hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
return error("%s: failed to insert into database",
path);
strbuf_release(&sb);
break;
case S_IFDIR:
return resolve_gitlink_ref(path, "HEAD", sha1);
default:
return error("%s: unsupported file type", path);
}
return 0;
}
int read_pack_header(int fd, struct pack_header *header)
{
if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
/* "eof before pack header was fully read" */
return PH_ERROR_EOF;
if (header->hdr_signature != htonl(PACK_SIGNATURE))
/* "protocol error (pack signature mismatch detected)" */
return PH_ERROR_PACK_SIGNATURE;
if (!pack_version_ok(header->hdr_version))
/* "protocol error (pack version unsupported)" */
return PH_ERROR_PROTOCOL;
return 0;
}
void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
{
enum object_type type = sha1_object_info(sha1, NULL);
if (type < 0)
die("%s is not a valid object", sha1_to_hex(sha1));
if (type != expect)
die("%s is not a valid '%s' object", sha1_to_hex(sha1),
typename(expect));
}