2005-06-29 01:21:02 +04:00
|
|
|
#ifndef PACK_H
|
|
|
|
#define PACK_H
|
|
|
|
|
2006-07-12 07:45:31 +04:00
|
|
|
#include "object.h"
|
2011-10-28 22:40:48 +04:00
|
|
|
#include "csum-file.h"
|
2005-06-29 01:21:02 +04:00
|
|
|
|
2018-11-10 08:49:07 +03:00
|
|
|
struct repository;
|
|
|
|
|
2005-06-29 01:21:02 +04:00
|
|
|
/*
|
|
|
|
* Packed object header
|
|
|
|
*/
|
|
|
|
#define PACK_SIGNATURE 0x5041434b /* "PACK" */
|
2006-10-15 10:37:41 +04:00
|
|
|
#define PACK_VERSION 2
|
2006-02-10 01:50:04 +03:00
|
|
|
#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3))
|
2005-06-29 01:21:02 +04:00
|
|
|
struct pack_header {
|
2007-01-17 11:07:23 +03:00
|
|
|
uint32_t hdr_signature;
|
|
|
|
uint32_t hdr_version;
|
|
|
|
uint32_t hdr_entries;
|
2005-06-29 01:21:02 +04:00
|
|
|
};
|
|
|
|
|
2007-01-18 04:43:57 +03:00
|
|
|
/*
|
2007-03-16 23:42:50 +03:00
|
|
|
* The first four bytes of index formats later than version 1 should
|
|
|
|
* start with this signature, as all older git binaries would find this
|
|
|
|
* value illegal and abort reading the file.
|
2007-01-18 04:43:57 +03:00
|
|
|
*
|
|
|
|
* This is the case because the number of objects in a packfile
|
|
|
|
* cannot exceed 1,431,660,000 as every object would need at least
|
2007-03-16 23:42:50 +03:00
|
|
|
* 3 bytes of data and the overall packfile cannot exceed 4 GiB with
|
|
|
|
* version 1 of the index file due to the offsets limited to 32 bits.
|
|
|
|
* Clearly the signature exceeds this maximum.
|
2007-01-18 04:43:57 +03:00
|
|
|
*
|
|
|
|
* Very old git binaries will also compare the first 4 bytes to the
|
|
|
|
* next 4 bytes in the index and abort with a "non-monotonic index"
|
|
|
|
* error if the second 4 byte word is smaller than the first 4
|
|
|
|
* byte word. This would be true in the proposed future index
|
|
|
|
* format as idx_signature would be greater than idx_version.
|
|
|
|
*/
|
|
|
|
#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */
|
|
|
|
|
2011-02-26 02:43:25 +03:00
|
|
|
struct pack_idx_option {
|
2011-02-03 04:29:01 +03:00
|
|
|
unsigned flags;
|
|
|
|
/* flag bits */
|
2011-11-17 10:04:13 +04:00
|
|
|
#define WRITE_IDX_VERIFY 01 /* verify only, do not write the idx file */
|
|
|
|
#define WRITE_IDX_STRICT 02
|
2021-01-26 02:37:18 +03:00
|
|
|
#define WRITE_REV 04
|
|
|
|
#define WRITE_REV_VERIFY 010
|
2011-02-03 04:29:01 +03:00
|
|
|
|
2011-02-26 02:43:25 +03:00
|
|
|
uint32_t version;
|
|
|
|
uint32_t off32_limit;
|
2011-02-26 03:55:26 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* List of offsets that would fit within off32_limit but
|
|
|
|
* need to be written out as 64-bit entity for byte-for-byte
|
|
|
|
* verification.
|
|
|
|
*/
|
|
|
|
int anomaly_alloc, anomaly_nr;
|
|
|
|
uint32_t *anomaly;
|
2011-02-26 02:43:25 +03:00
|
|
|
};
|
|
|
|
|
2019-04-29 11:28:14 +03:00
|
|
|
void reset_pack_idx_option(struct pack_idx_option *);
|
2007-06-01 23:18:05 +04:00
|
|
|
|
2007-03-16 23:42:50 +03:00
|
|
|
/*
|
|
|
|
* Packed object index header
|
|
|
|
*/
|
|
|
|
struct pack_idx_header {
|
|
|
|
uint32_t idx_signature;
|
|
|
|
uint32_t idx_version;
|
|
|
|
};
|
|
|
|
|
2007-06-01 23:18:05 +04:00
|
|
|
/*
|
|
|
|
* Common part of object structure used for write_idx_file
|
|
|
|
*/
|
|
|
|
struct pack_idx_entry {
|
2017-05-07 01:10:11 +03:00
|
|
|
struct object_id oid;
|
2007-06-01 23:18:05 +04:00
|
|
|
uint32_t crc32;
|
|
|
|
off_t offset;
|
|
|
|
};
|
|
|
|
|
2011-11-07 06:59:25 +04:00
|
|
|
|
2011-11-07 06:59:26 +04:00
|
|
|
struct progress;
|
fsck: use streaming interface for large blobs in pack
For blobs, we want to make sure the on-disk data is not corrupted
(i.e. can be inflated and produce the expected SHA-1). Blob content is
opaque, there's nothing else inside to check for.
For really large blobs, we may want to avoid unpacking the entire blob
in memory, just to check whether it produces the same SHA-1. On 32-bit
systems, we may not have enough virtual address space for such memory
allocation. And even on 64-bit where it's not a problem, allocating a
lot more memory could result in kicking other parts of systems to swap
file, generating lots of I/O and slowing everything down.
For this particular operation, not unpacking the blob and letting
check_sha1_signature, which supports streaming interface, do the job
is sufficient. check_sha1_signature() is not shown in the diff,
unfortunately. But if will be called when "data_valid && !data" is
false.
We will call the callback function "fn" with NULL as "data". The only
callback of this function is fsck_obj_buffer(), which does not touch
"data" at all if it's a blob.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-07-13 18:44:04 +03:00
|
|
|
/* Note, the data argument could be NULL if object type is blob */
|
2017-05-07 01:10:20 +03:00
|
|
|
typedef int (*verify_fn)(const struct object_id *, enum object_type, unsigned long, void*, int*);
|
2011-11-07 06:59:25 +04:00
|
|
|
|
2019-04-29 11:28:14 +03:00
|
|
|
const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, const struct pack_idx_option *, const unsigned char *sha1);
|
|
|
|
int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr);
|
|
|
|
int verify_pack_index(struct packed_git *);
|
|
|
|
int verify_pack(struct repository *, struct packed_git *, verify_fn fn, struct progress *, uint32_t);
|
|
|
|
off_t write_pack_header(struct hashfile *f, uint32_t);
|
|
|
|
void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t, unsigned char *, off_t);
|
2021-02-22 22:20:09 +03:00
|
|
|
char *index_pack_lockfile(int fd, int *is_well_formed);
|
2017-03-24 20:26:50 +03:00
|
|
|
|
2021-01-12 11:21:59 +03:00
|
|
|
struct ref;
|
|
|
|
|
|
|
|
void write_promisor_file(const char *promisor_name, struct ref **sought, int nr_sought);
|
|
|
|
|
2021-01-26 02:37:18 +03:00
|
|
|
const char *write_rev_file(const char *rev_name, struct pack_idx_entry **objects, uint32_t nr_objects, const unsigned char *hash, unsigned flags);
|
2021-03-30 18:04:29 +03:00
|
|
|
const char *write_rev_file_order(const char *rev_name, uint32_t *pack_order, uint32_t nr_objects, const unsigned char *hash, unsigned flags);
|
2021-01-26 02:37:18 +03:00
|
|
|
|
2017-03-24 20:26:50 +03:00
|
|
|
/*
|
|
|
|
* The "hdr" output buffer should be at least this big, which will handle sizes
|
|
|
|
* up to 2^67.
|
|
|
|
*/
|
|
|
|
#define MAX_PACK_OBJECT_HEADER 10
|
2019-04-29 11:28:14 +03:00
|
|
|
int encode_in_pack_object_header(unsigned char *hdr, int hdr_len,
|
2019-04-29 11:28:23 +03:00
|
|
|
enum object_type, uintmax_t);
|
2007-01-23 08:55:18 +03:00
|
|
|
|
|
|
|
#define PH_ERROR_EOF (-1)
|
|
|
|
#define PH_ERROR_PACK_SIGNATURE (-2)
|
|
|
|
#define PH_ERROR_PROTOCOL (-3)
|
2019-04-29 11:28:14 +03:00
|
|
|
int read_pack_header(int fd, struct pack_header *);
|
2011-10-28 22:52:14 +04:00
|
|
|
|
2019-04-29 11:28:14 +03:00
|
|
|
struct hashfile *create_tmp_packfile(char **pack_tmp_name);
|
2021-09-10 02:24:56 +03:00
|
|
|
void stage_tmp_packfiles(struct strbuf *name_buffer,
|
2021-09-09 06:24:47 +03:00
|
|
|
const char *pack_tmp_name,
|
|
|
|
struct pack_idx_entry **written_list,
|
|
|
|
uint32_t nr_written,
|
|
|
|
struct pack_idx_option *pack_idx_opts,
|
2021-09-10 02:24:56 +03:00
|
|
|
unsigned char hash[],
|
|
|
|
char **idx_tmp_name);
|
|
|
|
void rename_tmp_packfile_idx(struct strbuf *basename,
|
|
|
|
char **idx_tmp_name);
|
2011-10-28 22:52:14 +04:00
|
|
|
|
2005-06-29 01:21:02 +04:00
|
|
|
#endif
|