2005-12-05 22:54:29 +03:00
|
|
|
#ifndef GIT_COMPAT_UTIL_H
|
|
|
|
#define GIT_COMPAT_UTIL_H
|
|
|
|
|
2007-02-17 12:13:10 +03:00
|
|
|
#define _FILE_OFFSET_BITS 64
|
|
|
|
|
2006-01-07 12:33:54 +03:00
|
|
|
#ifndef FLEX_ARRAY
|
2007-11-20 23:08:06 +03:00
|
|
|
/*
|
|
|
|
* See if our compiler is known to support flexible array members.
|
|
|
|
*/
|
|
|
|
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
|
|
|
|
# define FLEX_ARRAY /* empty */
|
|
|
|
#elif defined(__GNUC__)
|
|
|
|
# if (__GNUC__ >= 3)
|
|
|
|
# define FLEX_ARRAY /* empty */
|
|
|
|
# else
|
|
|
|
# define FLEX_ARRAY 0 /* older GNU extension */
|
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Otherwise, default to safer but a bit wasteful traditional style
|
|
|
|
*/
|
|
|
|
#ifndef FLEX_ARRAY
|
|
|
|
# define FLEX_ARRAY 1
|
2006-01-07 12:33:54 +03:00
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2006-03-09 22:58:05 +03:00
|
|
|
#define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
|
|
|
|
|
2007-04-09 09:06:29 +04:00
|
|
|
#ifdef __GNUC__
|
|
|
|
#define TYPEOF(x) (__typeof__(x))
|
|
|
|
#else
|
|
|
|
#define TYPEOF(x)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define MSB(x, bits) ((x) & TYPEOF(x)(~0ULL << (sizeof(x) * 8 - (bits))))
|
2007-11-07 13:20:27 +03:00
|
|
|
#define HAS_MULTI_BITS(i) ((i) & ((i) - 1)) /* checks if an integer has more than 1 bit set */
|
2007-04-09 09:06:29 +04:00
|
|
|
|
2007-05-15 20:33:25 +04:00
|
|
|
/* Approximation of the length of the decimal representation of this type. */
|
|
|
|
#define decimal_length(x) ((int)(sizeof(x) * 2.56 + 0.5) + 1)
|
|
|
|
|
2006-12-23 09:46:11 +03:00
|
|
|
#if !defined(__APPLE__) && !defined(__FreeBSD__)
|
2006-12-20 01:34:12 +03:00
|
|
|
#define _XOPEN_SOURCE 600 /* glibc2 and AIX 5.3L need 500, OpenBSD needs 600 for S_ISLNK() */
|
|
|
|
#define _XOPEN_SOURCE_EXTENDED 1 /* AIX 5.3L needs this */
|
2006-12-21 04:32:21 +03:00
|
|
|
#endif
|
2007-01-16 04:34:49 +03:00
|
|
|
#define _ALL_SOURCE 1
|
|
|
|
#define _GNU_SOURCE 1
|
|
|
|
#define _BSD_SOURCE 1
|
2006-12-20 01:34:12 +03:00
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#include <unistd.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <stdarg.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <errno.h>
|
|
|
|
#include <limits.h>
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <dirent.h>
|
2006-12-20 01:34:12 +03:00
|
|
|
#include <sys/time.h>
|
|
|
|
#include <time.h>
|
|
|
|
#include <signal.h>
|
|
|
|
#include <sys/wait.h>
|
|
|
|
#include <fnmatch.h>
|
|
|
|
#include <sys/poll.h>
|
|
|
|
#include <sys/socket.h>
|
|
|
|
#include <assert.h>
|
|
|
|
#include <regex.h>
|
|
|
|
#include <netinet/in.h>
|
|
|
|
#include <netinet/tcp.h>
|
|
|
|
#include <arpa/inet.h>
|
|
|
|
#include <netdb.h>
|
|
|
|
#include <pwd.h>
|
2007-01-26 00:11:40 +03:00
|
|
|
#include <inttypes.h>
|
2007-03-03 21:28:52 +03:00
|
|
|
#if defined(__CYGWIN__)
|
|
|
|
#undef _XOPEN_SOURCE
|
|
|
|
#include <grp.h>
|
|
|
|
#define _XOPEN_SOURCE 600
|
|
|
|
#else
|
2007-01-16 04:34:49 +03:00
|
|
|
#undef _ALL_SOURCE /* AIX 5.3L defines a struct list with _ALL_SOURCE. */
|
2006-12-20 01:34:12 +03:00
|
|
|
#include <grp.h>
|
2007-01-16 04:34:49 +03:00
|
|
|
#define _ALL_SOURCE 1
|
2007-03-03 21:28:52 +03:00
|
|
|
#endif
|
2006-12-20 01:34:12 +03:00
|
|
|
|
|
|
|
#ifndef NO_ICONV
|
|
|
|
#include <iconv.h>
|
|
|
|
#endif
|
2005-12-05 22:54:29 +03:00
|
|
|
|
2006-09-16 09:47:21 +04:00
|
|
|
/* On most systems <limits.h> would have given us this, but
|
|
|
|
* not on some systems (e.g. GNU/Hurd).
|
|
|
|
*/
|
|
|
|
#ifndef PATH_MAX
|
|
|
|
#define PATH_MAX 4096
|
|
|
|
#endif
|
|
|
|
|
2007-03-07 04:44:30 +03:00
|
|
|
#ifndef PRIuMAX
|
|
|
|
#define PRIuMAX "llu"
|
|
|
|
#endif
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#ifdef __GNUC__
|
|
|
|
#define NORETURN __attribute__((__noreturn__))
|
|
|
|
#else
|
|
|
|
#define NORETURN
|
|
|
|
#ifndef __attribute__
|
|
|
|
#define __attribute__(x)
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* General helper functions */
|
|
|
|
extern void usage(const char *err) NORETURN;
|
|
|
|
extern void die(const char *err, ...) NORETURN __attribute__((format (printf, 1, 2)));
|
|
|
|
extern int error(const char *err, ...) __attribute__((format (printf, 1, 2)));
|
2007-03-31 03:07:05 +04:00
|
|
|
extern void warning(const char *err, ...) __attribute__((format (printf, 1, 2)));
|
2005-12-05 22:54:29 +03:00
|
|
|
|
2006-06-24 06:34:38 +04:00
|
|
|
extern void set_usage_routine(void (*routine)(const char *err) NORETURN);
|
|
|
|
extern void set_die_routine(void (*routine)(const char *err, va_list params) NORETURN);
|
|
|
|
extern void set_error_routine(void (*routine)(const char *err, va_list params));
|
2006-12-22 03:48:32 +03:00
|
|
|
extern void set_warn_routine(void (*routine)(const char *warn, va_list params));
|
2006-06-24 06:34:38 +04:00
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#ifdef NO_MMAP
|
|
|
|
|
|
|
|
#ifndef PROT_READ
|
|
|
|
#define PROT_READ 1
|
|
|
|
#define PROT_WRITE 2
|
|
|
|
#define MAP_PRIVATE 1
|
|
|
|
#define MAP_FAILED ((void*)-1)
|
|
|
|
#endif
|
|
|
|
|
2006-12-24 08:45:37 +03:00
|
|
|
#define mmap git_mmap
|
|
|
|
#define munmap git_munmap
|
|
|
|
extern void *git_mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
|
|
|
|
extern int git_munmap(void *start, size_t length);
|
2005-12-05 22:54:29 +03:00
|
|
|
|
2007-02-15 00:20:41 +03:00
|
|
|
/* This value must be multiple of (pagesize * 2) */
|
2006-12-24 08:46:13 +03:00
|
|
|
#define DEFAULT_PACKED_GIT_WINDOW_SIZE (1 * 1024 * 1024)
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#else /* NO_MMAP */
|
|
|
|
|
|
|
|
#include <sys/mman.h>
|
2007-02-15 00:20:41 +03:00
|
|
|
|
|
|
|
/* This value must be multiple of (pagesize * 2) */
|
2007-01-05 06:28:08 +03:00
|
|
|
#define DEFAULT_PACKED_GIT_WINDOW_SIZE \
|
|
|
|
(sizeof(void*) >= 8 \
|
|
|
|
? 1 * 1024 * 1024 * 1024 \
|
|
|
|
: 32 * 1024 * 1024)
|
2005-12-05 22:54:29 +03:00
|
|
|
|
|
|
|
#endif /* NO_MMAP */
|
|
|
|
|
2007-01-05 06:28:08 +03:00
|
|
|
#define DEFAULT_PACKED_GIT_LIMIT \
|
2007-01-07 11:11:11 +03:00
|
|
|
((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256))
|
2006-12-24 08:46:13 +03:00
|
|
|
|
2007-01-10 00:04:12 +03:00
|
|
|
#ifdef NO_PREAD
|
|
|
|
#define pread git_pread
|
|
|
|
extern ssize_t git_pread(int fd, void *buf, size_t count, off_t offset);
|
|
|
|
#endif
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#ifdef NO_SETENV
|
|
|
|
#define setenv gitsetenv
|
|
|
|
extern int gitsetenv(const char *, const char *, int);
|
|
|
|
#endif
|
|
|
|
|
2007-10-21 00:03:49 +04:00
|
|
|
#ifdef NO_MKDTEMP
|
|
|
|
#define mkdtemp gitmkdtemp
|
|
|
|
extern char *gitmkdtemp(char *);
|
|
|
|
#endif
|
|
|
|
|
2006-01-25 23:38:36 +03:00
|
|
|
#ifdef NO_UNSETENV
|
|
|
|
#define unsetenv gitunsetenv
|
|
|
|
extern void gitunsetenv(const char *);
|
|
|
|
#endif
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#ifdef NO_STRCASESTR
|
|
|
|
#define strcasestr gitstrcasestr
|
|
|
|
extern char *gitstrcasestr(const char *haystack, const char *needle);
|
|
|
|
#endif
|
|
|
|
|
2006-06-24 18:01:25 +04:00
|
|
|
#ifdef NO_STRLCPY
|
|
|
|
#define strlcpy gitstrlcpy
|
|
|
|
extern size_t gitstrlcpy(char *, const char *, size_t);
|
|
|
|
#endif
|
|
|
|
|
2007-02-20 03:22:56 +03:00
|
|
|
#ifdef NO_STRTOUMAX
|
|
|
|
#define strtoumax gitstrtoumax
|
|
|
|
extern uintmax_t gitstrtoumax(const char *, char **, int);
|
|
|
|
#endif
|
|
|
|
|
2007-06-13 22:54:32 +04:00
|
|
|
#ifdef NO_HSTRERROR
|
|
|
|
#define hstrerror githstrerror
|
|
|
|
extern const char *githstrerror(int herror);
|
|
|
|
#endif
|
|
|
|
|
2007-09-07 02:32:54 +04:00
|
|
|
#ifdef NO_MEMMEM
|
|
|
|
#define memmem gitmemmem
|
|
|
|
void *gitmemmem(const void *haystack, size_t haystacklen,
|
|
|
|
const void *needle, size_t needlelen);
|
|
|
|
#endif
|
|
|
|
|
2007-11-12 13:09:05 +03:00
|
|
|
#ifdef __GLIBC_PREREQ
|
|
|
|
#if __GLIBC_PREREQ(2, 1)
|
|
|
|
#define HAVE_STRCHRNUL
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef HAVE_STRCHRNUL
|
2007-11-09 03:49:36 +03:00
|
|
|
#define strchrnul gitstrchrnul
|
2007-11-10 14:55:48 +03:00
|
|
|
static inline char *gitstrchrnul(const char *s, int c)
|
|
|
|
{
|
|
|
|
while (*s && *s != c)
|
|
|
|
s++;
|
|
|
|
return (char *)s;
|
|
|
|
}
|
2007-11-09 03:49:36 +03:00
|
|
|
#endif
|
|
|
|
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
extern void release_pack_memory(size_t, int);
|
2006-12-24 08:47:19 +03:00
|
|
|
|
2006-09-02 08:16:31 +04:00
|
|
|
static inline char* xstrdup(const char *str)
|
|
|
|
{
|
|
|
|
char *ret = strdup(str);
|
2006-12-24 08:47:19 +03:00
|
|
|
if (!ret) {
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
release_pack_memory(strlen(str) + 1, -1);
|
2006-12-24 08:47:19 +03:00
|
|
|
ret = strdup(str);
|
|
|
|
if (!ret)
|
|
|
|
die("Out of memory, strdup failed");
|
|
|
|
}
|
2006-09-02 08:16:31 +04:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
static inline void *xmalloc(size_t size)
|
|
|
|
{
|
|
|
|
void *ret = malloc(size);
|
2005-12-29 12:31:26 +03:00
|
|
|
if (!ret && !size)
|
|
|
|
ret = malloc(1);
|
2006-12-24 08:47:19 +03:00
|
|
|
if (!ret) {
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
release_pack_memory(size, -1);
|
2006-12-24 08:47:19 +03:00
|
|
|
ret = malloc(size);
|
|
|
|
if (!ret && !size)
|
|
|
|
ret = malloc(1);
|
|
|
|
if (!ret)
|
|
|
|
die("Out of memory, malloc failed");
|
|
|
|
}
|
2006-08-08 23:23:47 +04:00
|
|
|
#ifdef XMALLOC_POISON
|
|
|
|
memset(ret, 0xA5, size);
|
|
|
|
#endif
|
2005-12-05 22:54:29 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2007-09-16 01:53:05 +04:00
|
|
|
static inline void *xmemdupz(const void *data, size_t len)
|
2007-05-03 06:49:41 +04:00
|
|
|
{
|
2007-09-16 01:53:05 +04:00
|
|
|
char *p = xmalloc(len + 1);
|
|
|
|
memcpy(p, data, len);
|
2007-05-03 06:49:41 +04:00
|
|
|
p[len] = '\0';
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2007-09-16 01:53:05 +04:00
|
|
|
static inline char *xstrndup(const char *str, size_t len)
|
|
|
|
{
|
|
|
|
char *p = memchr(str, '\0', len);
|
|
|
|
return xmemdupz(str, p ? p - str : len);
|
|
|
|
}
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
static inline void *xrealloc(void *ptr, size_t size)
|
|
|
|
{
|
|
|
|
void *ret = realloc(ptr, size);
|
2005-12-29 12:31:26 +03:00
|
|
|
if (!ret && !size)
|
|
|
|
ret = realloc(ptr, 1);
|
2006-12-24 08:47:19 +03:00
|
|
|
if (!ret) {
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
release_pack_memory(size, -1);
|
2006-12-24 08:47:19 +03:00
|
|
|
ret = realloc(ptr, size);
|
|
|
|
if (!ret && !size)
|
|
|
|
ret = realloc(ptr, 1);
|
|
|
|
if (!ret)
|
|
|
|
die("Out of memory, realloc failed");
|
|
|
|
}
|
2005-12-05 22:54:29 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void *xcalloc(size_t nmemb, size_t size)
|
|
|
|
{
|
|
|
|
void *ret = calloc(nmemb, size);
|
2005-12-29 12:31:26 +03:00
|
|
|
if (!ret && (!nmemb || !size))
|
|
|
|
ret = calloc(1, 1);
|
2006-12-24 08:47:19 +03:00
|
|
|
if (!ret) {
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
release_pack_memory(nmemb * size, -1);
|
2006-12-24 08:47:19 +03:00
|
|
|
ret = calloc(nmemb, size);
|
|
|
|
if (!ret && (!nmemb || !size))
|
|
|
|
ret = calloc(1, 1);
|
|
|
|
if (!ret)
|
|
|
|
die("Out of memory, calloc failed");
|
|
|
|
}
|
2005-12-05 22:54:29 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2006-12-24 08:47:23 +03:00
|
|
|
static inline void *xmmap(void *start, size_t length,
|
|
|
|
int prot, int flags, int fd, off_t offset)
|
|
|
|
{
|
|
|
|
void *ret = mmap(start, length, prot, flags, fd, offset);
|
|
|
|
if (ret == MAP_FAILED) {
|
Better error messages for corrupt databases
This fixes another problem that Andy's case showed: git-fsck-objects
reports nonsensical results for corrupt objects.
There were actually two independent and confusing problems:
- when we had a zero-sized file and used map_sha1_file, mmap() would
return EINVAL, and git-fsck-objects would report that as an insane and
confusing error. I don't know when this was introduced, it might have
been there forever.
- when "parse_object()" returned NULL, fsck would say "object not found",
which can be very confusing, since obviously the object might "exist",
it's just unparseable because it's totally corrupt.
So this just makes "xmmap()" return NULL for a zero-sized object (which is
a valid thing pointer, exactly the same way "malloc()" can return NULL for
a zero-sized allocation). That fixes the first problem (but we could have
fixed it in the caller too - I don't personally much care whichever way it
goes, but maybe somebody should check that the NO_MMAP case does
something sane in this case too?).
And the second problem is solved by just making the error message slightly
clearer - the failure to parse an object may be because it's missing or
corrupt, not necessarily because it's not "found".
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-01-12 01:09:31 +03:00
|
|
|
if (!length)
|
|
|
|
return NULL;
|
Actually handle some-low memory conditions
Tim Ansell discovered his Debian server didn't permit git-daemon to
use as much memory as it needed to handle cloning a project with
a 128 MiB packfile. Filtering the strace provided by Tim of the
rev-list child showed this gem of a sequence:
open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...>
<... open resumed> ) = 5
OK, so the packfile is fd 5...
mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...>
<... mmap2 resumed> ) = 0xb5e2d000
and we mapped one 32 MiB window from it at position 0...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory)
And we asked for another window further into the file. But got
denied. In Tim's case this was due to a resource limit on the
git-daemon process, and its children.
Now where are we in the code? We're down inside use_pack(),
after we have called unuse_one_window() enough times to make sure
we stay within our allowed maximum window size. However since we
didn't unmap the prior window at 0xb5e2d000 we aren't exceeding
the current limit (which probably was just the defaults).
But we're actually down inside xmmap()...
So we release the window we do have (by calling release_pack_memory),
assuming there is some memory pressure...
munmap(0xb5e2d000, 33554432 <unfinished ...>
<... munmap resumed> ) = 0
close(5 <unfinished ...>
<... close resumed> ) = 0
And that was the last window in this packfile. So we closed it.
Way to go us. Our xmmap did not expect release_pack_memory to
close the fd its about to map...
mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...>
<... mmap2 resumed> ) = -1 EBADF (Bad file descriptor)
And so the Linux kernel happily tells us f' off.
write(2, "fatal: ", 7 <unfinished ...>
<... write resumed> ) = 7
write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...>
<... write resumed> ) = 47
And we report the bad file descriptor error, and not the ENOMEM,
and die, claiming we are out of memory. But actually that mmap
should have succeeded, as we had enough memory for that window,
seeing as how we released the prior one.
Originally when I developed the sliding window mmap feature I had
this exact same bug in fast-import, and I dealt with it by handing
in the struct packed_git* we want to open the new window for, as the
caller wasn't prepared to reopen the packfile if unuse_one_window
closed it. The same is true here from xmmap, but the caller doesn't
have the struct packed_git* handy. So I'm using the file descriptor
instead to perform the same test.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
2007-04-25 12:02:27 +04:00
|
|
|
release_pack_memory(length, fd);
|
2006-12-24 08:47:23 +03:00
|
|
|
ret = mmap(start, length, prot, flags, fd, offset);
|
|
|
|
if (ret == MAP_FAILED)
|
|
|
|
die("Out of memory? mmap failed: %s", strerror(errno));
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2005-12-20 03:18:28 +03:00
|
|
|
static inline ssize_t xread(int fd, void *buf, size_t len)
|
|
|
|
{
|
|
|
|
ssize_t nr;
|
|
|
|
while (1) {
|
|
|
|
nr = read(fd, buf, len);
|
|
|
|
if ((nr < 0) && (errno == EAGAIN || errno == EINTR))
|
|
|
|
continue;
|
|
|
|
return nr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline ssize_t xwrite(int fd, const void *buf, size_t len)
|
|
|
|
{
|
|
|
|
ssize_t nr;
|
|
|
|
while (1) {
|
|
|
|
nr = write(fd, buf, len);
|
|
|
|
if ((nr < 0) && (errno == EAGAIN || errno == EINTR))
|
|
|
|
continue;
|
|
|
|
return nr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-06-27 18:28:53 +04:00
|
|
|
static inline int xdup(int fd)
|
|
|
|
{
|
|
|
|
int ret = dup(fd);
|
|
|
|
if (ret < 0)
|
|
|
|
die("dup failed: %s", strerror(errno));
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline FILE *xfdopen(int fd, const char *mode)
|
|
|
|
{
|
|
|
|
FILE *stream = fdopen(fd, mode);
|
|
|
|
if (stream == NULL)
|
|
|
|
die("Out of memory? fdopen failed: %s", strerror(errno));
|
|
|
|
return stream;
|
|
|
|
}
|
|
|
|
|
2007-08-14 23:44:53 +04:00
|
|
|
static inline int xmkstemp(char *template)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
|
|
|
|
fd = mkstemp(template);
|
|
|
|
if (fd < 0)
|
|
|
|
die("Unable to create temporary file: %s", strerror(errno));
|
|
|
|
return fd;
|
|
|
|
}
|
|
|
|
|
2007-03-07 04:44:37 +03:00
|
|
|
static inline size_t xsize_t(off_t len)
|
|
|
|
{
|
|
|
|
return (size_t)len;
|
|
|
|
}
|
|
|
|
|
2006-08-11 16:01:45 +04:00
|
|
|
static inline int has_extension(const char *filename, const char *ext)
|
2006-08-10 19:02:30 +04:00
|
|
|
{
|
2006-08-11 16:01:45 +04:00
|
|
|
size_t len = strlen(filename);
|
|
|
|
size_t extlen = strlen(ext);
|
2006-08-10 19:02:30 +04:00
|
|
|
return len > extlen && !memcmp(filename + len - extlen, ext, extlen);
|
|
|
|
}
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
/* Sane ctype - no locale, and works with signed chars */
|
|
|
|
#undef isspace
|
|
|
|
#undef isdigit
|
|
|
|
#undef isalpha
|
|
|
|
#undef isalnum
|
|
|
|
#undef tolower
|
|
|
|
#undef toupper
|
|
|
|
extern unsigned char sane_ctype[256];
|
|
|
|
#define GIT_SPACE 0x01
|
|
|
|
#define GIT_DIGIT 0x02
|
|
|
|
#define GIT_ALPHA 0x04
|
|
|
|
#define sane_istest(x,mask) ((sane_ctype[(unsigned char)(x)] & (mask)) != 0)
|
|
|
|
#define isspace(x) sane_istest(x,GIT_SPACE)
|
|
|
|
#define isdigit(x) sane_istest(x,GIT_DIGIT)
|
|
|
|
#define isalpha(x) sane_istest(x,GIT_ALPHA)
|
|
|
|
#define isalnum(x) sane_istest(x,GIT_ALPHA | GIT_DIGIT)
|
|
|
|
#define tolower(x) sane_case((unsigned char)(x), 0x20)
|
|
|
|
#define toupper(x) sane_case((unsigned char)(x), 0)
|
|
|
|
|
|
|
|
static inline int sane_case(int x, int high)
|
|
|
|
{
|
|
|
|
if (sane_istest(x, GIT_ALPHA))
|
|
|
|
x = (x & ~0x20) | high;
|
|
|
|
return x;
|
|
|
|
}
|
|
|
|
|
2007-02-20 12:51:22 +03:00
|
|
|
static inline int prefixcmp(const char *str, const char *prefix)
|
|
|
|
{
|
|
|
|
return strncmp(str, prefix, strlen(prefix));
|
|
|
|
}
|
|
|
|
|
2007-04-10 03:01:44 +04:00
|
|
|
static inline int strtoul_ui(char const *s, int base, unsigned int *result)
|
|
|
|
{
|
|
|
|
unsigned long ul;
|
|
|
|
char *p;
|
|
|
|
|
|
|
|
errno = 0;
|
|
|
|
ul = strtoul(s, &p, base);
|
|
|
|
if (errno || *p || p == s || (unsigned int) ul != ul)
|
|
|
|
return -1;
|
|
|
|
*result = ul;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-10-24 00:33:26 +04:00
|
|
|
static inline int strtol_i(char const *s, int base, int *result)
|
|
|
|
{
|
|
|
|
long ul;
|
|
|
|
char *p;
|
|
|
|
|
|
|
|
errno = 0;
|
|
|
|
ul = strtol(s, &p, base);
|
|
|
|
if (errno || *p || p == s || (int) ul != ul)
|
|
|
|
return -1;
|
|
|
|
*result = ul;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-12-05 22:54:29 +03:00
|
|
|
#endif
|