git/reftable/block.c

453 строки
10 KiB
C

/*
Copyright 2020 Google LLC
Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
#include "block.h"
#include "blocksource.h"
#include "constants.h"
#include "record.h"
#include "reftable-error.h"
#include "system.h"
#include <zlib.h>
int header_size(int version)
{
switch (version) {
case 1:
return 24;
case 2:
return 28;
}
abort();
}
int footer_size(int version)
{
switch (version) {
case 1:
return 68;
case 2:
return 72;
}
abort();
}
static int block_writer_register_restart(struct block_writer *w, int n,
int is_restart, struct strbuf *key)
{
int rlen = w->restart_len;
if (rlen >= MAX_RESTARTS) {
is_restart = 0;
}
if (is_restart) {
rlen++;
}
if (2 + 3 * rlen + n > w->block_size - w->next)
return -1;
if (is_restart) {
if (w->restart_len == w->restart_cap) {
w->restart_cap = w->restart_cap * 2 + 1;
w->restarts = reftable_realloc(
w->restarts, sizeof(uint32_t) * w->restart_cap);
}
w->restarts[w->restart_len++] = w->next;
}
w->next += n;
strbuf_reset(&w->last_key);
strbuf_addbuf(&w->last_key, key);
w->entries++;
return 0;
}
void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf,
uint32_t block_size, uint32_t header_off, int hash_size)
{
bw->buf = buf;
bw->hash_size = hash_size;
bw->block_size = block_size;
bw->header_off = header_off;
bw->buf[header_off] = typ;
bw->next = header_off + 4;
bw->restart_interval = 16;
bw->entries = 0;
bw->restart_len = 0;
bw->last_key.len = 0;
}
uint8_t block_writer_type(struct block_writer *bw)
{
return bw->buf[bw->header_off];
}
/* Adds the reftable_record to the block. Returns -1 if it does not fit, 0 on
success. Returns REFTABLE_API_ERROR if attempting to write a record with
empty key. */
int block_writer_add(struct block_writer *w, struct reftable_record *rec)
{
struct strbuf empty = STRBUF_INIT;
struct strbuf last =
w->entries % w->restart_interval == 0 ? empty : w->last_key;
struct string_view out = {
.buf = w->buf + w->next,
.len = w->block_size - w->next,
};
struct string_view start = out;
int is_restart = 0;
struct strbuf key = STRBUF_INIT;
int n = 0;
int err = -1;
reftable_record_key(rec, &key);
if (!key.len) {
err = REFTABLE_API_ERROR;
goto done;
}
n = reftable_encode_key(&is_restart, out, last, key,
reftable_record_val_type(rec));
if (n < 0)
goto done;
string_view_consume(&out, n);
n = reftable_record_encode(rec, out, w->hash_size);
if (n < 0)
goto done;
string_view_consume(&out, n);
err = block_writer_register_restart(w, start.len - out.len, is_restart,
&key);
done:
strbuf_release(&key);
return err;
}
int block_writer_finish(struct block_writer *w)
{
int i;
for (i = 0; i < w->restart_len; i++) {
put_be24(w->buf + w->next, w->restarts[i]);
w->next += 3;
}
put_be16(w->buf + w->next, w->restart_len);
w->next += 2;
put_be24(w->buf + 1 + w->header_off, w->next);
if (block_writer_type(w) == BLOCK_TYPE_LOG) {
int block_header_skip = 4 + w->header_off;
uLongf src_len = w->next - block_header_skip;
uLongf dest_cap = src_len * 1.001 + 12;
uint8_t *compressed = reftable_malloc(dest_cap);
while (1) {
uLongf out_dest_len = dest_cap;
int zresult = compress2(compressed, &out_dest_len,
w->buf + block_header_skip,
src_len, 9);
if (zresult == Z_BUF_ERROR && dest_cap < LONG_MAX) {
dest_cap *= 2;
compressed =
reftable_realloc(compressed, dest_cap);
if (compressed)
continue;
}
if (Z_OK != zresult) {
reftable_free(compressed);
return REFTABLE_ZLIB_ERROR;
}
memcpy(w->buf + block_header_skip, compressed,
out_dest_len);
w->next = out_dest_len + block_header_skip;
reftable_free(compressed);
break;
}
}
return w->next;
}
uint8_t block_reader_type(struct block_reader *r)
{
return r->block.data[r->header_off];
}
int block_reader_init(struct block_reader *br, struct reftable_block *block,
uint32_t header_off, uint32_t table_block_size,
int hash_size)
{
uint32_t full_block_size = table_block_size;
uint8_t typ = block->data[header_off];
uint32_t sz = get_be24(block->data + header_off + 1);
int err = 0;
uint16_t restart_count = 0;
uint32_t restart_start = 0;
uint8_t *restart_bytes = NULL;
uint8_t *uncompressed = NULL;
if (!reftable_is_block_type(typ)) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}
if (typ == BLOCK_TYPE_LOG) {
int block_header_skip = 4 + header_off;
uLongf dst_len = sz - block_header_skip; /* total size of dest
buffer. */
uLongf src_len = block->len - block_header_skip;
/* Log blocks specify the *uncompressed* size in their header.
*/
uncompressed = reftable_malloc(sz);
/* Copy over the block header verbatim. It's not compressed. */
memcpy(uncompressed, block->data, block_header_skip);
/* Uncompress */
if (Z_OK !=
uncompress2(uncompressed + block_header_skip, &dst_len,
block->data + block_header_skip, &src_len)) {
err = REFTABLE_ZLIB_ERROR;
goto done;
}
if (dst_len + block_header_skip != sz) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}
/* We're done with the input data. */
reftable_block_done(block);
block->data = uncompressed;
uncompressed = NULL;
block->len = sz;
block->source = malloc_block_source();
full_block_size = src_len + block_header_skip;
} else if (full_block_size == 0) {
full_block_size = sz;
} else if (sz < full_block_size && sz < block->len &&
block->data[sz] != 0) {
/* If the block is smaller than the full block size, it is
padded (data followed by '\0') or the next block is
unaligned. */
full_block_size = sz;
}
restart_count = get_be16(block->data + sz - 2);
restart_start = sz - 2 - 3 * restart_count;
restart_bytes = block->data + restart_start;
/* transfer ownership. */
br->block = *block;
block->data = NULL;
block->len = 0;
br->hash_size = hash_size;
br->block_len = restart_start;
br->full_block_size = full_block_size;
br->header_off = header_off;
br->restart_count = restart_count;
br->restart_bytes = restart_bytes;
done:
reftable_free(uncompressed);
return err;
}
static uint32_t block_reader_restart_offset(struct block_reader *br, int i)
{
return get_be24(br->restart_bytes + 3 * i);
}
void block_reader_start(struct block_reader *br, struct block_iter *it)
{
it->br = br;
strbuf_reset(&it->last_key);
it->next_off = br->header_off + 4;
}
struct restart_find_args {
int error;
struct strbuf key;
struct block_reader *r;
};
static int restart_key_less(size_t idx, void *args)
{
struct restart_find_args *a = args;
uint32_t off = block_reader_restart_offset(a->r, idx);
struct string_view in = {
.buf = a->r->block.data + off,
.len = a->r->block_len - off,
};
/* the restart key is verbatim in the block, so this could avoid the
alloc for decoding the key */
struct strbuf rkey = STRBUF_INIT;
struct strbuf last_key = STRBUF_INIT;
uint8_t unused_extra;
int n = reftable_decode_key(&rkey, &unused_extra, last_key, in);
int result;
if (n < 0) {
a->error = 1;
return -1;
}
result = strbuf_cmp(&a->key, &rkey);
strbuf_release(&rkey);
return result;
}
void block_iter_copy_from(struct block_iter *dest, struct block_iter *src)
{
dest->br = src->br;
dest->next_off = src->next_off;
strbuf_reset(&dest->last_key);
strbuf_addbuf(&dest->last_key, &src->last_key);
}
int block_iter_next(struct block_iter *it, struct reftable_record *rec)
{
struct string_view in = {
.buf = it->br->block.data + it->next_off,
.len = it->br->block_len - it->next_off,
};
struct string_view start = in;
struct strbuf key = STRBUF_INIT;
uint8_t extra = 0;
int n = 0;
if (it->next_off >= it->br->block_len)
return 1;
n = reftable_decode_key(&key, &extra, it->last_key, in);
if (n < 0)
return -1;
if (!key.len)
return REFTABLE_FORMAT_ERROR;
string_view_consume(&in, n);
n = reftable_record_decode(rec, key, extra, in, it->br->hash_size);
if (n < 0)
return -1;
string_view_consume(&in, n);
strbuf_reset(&it->last_key);
strbuf_addbuf(&it->last_key, &key);
it->next_off += start.len - in.len;
strbuf_release(&key);
return 0;
}
int block_reader_first_key(struct block_reader *br, struct strbuf *key)
{
struct strbuf empty = STRBUF_INIT;
int off = br->header_off + 4;
struct string_view in = {
.buf = br->block.data + off,
.len = br->block_len - off,
};
uint8_t extra = 0;
int n = reftable_decode_key(key, &extra, empty, in);
if (n < 0)
return n;
if (!key->len)
return REFTABLE_FORMAT_ERROR;
return 0;
}
int block_iter_seek(struct block_iter *it, struct strbuf *want)
{
return block_reader_seek(it->br, it, want);
}
void block_iter_close(struct block_iter *it)
{
strbuf_release(&it->last_key);
}
int block_reader_seek(struct block_reader *br, struct block_iter *it,
struct strbuf *want)
{
struct restart_find_args args = {
.key = *want,
.r = br,
};
struct reftable_record rec = reftable_new_record(block_reader_type(br));
struct strbuf key = STRBUF_INIT;
int err = 0;
struct block_iter next = {
.last_key = STRBUF_INIT,
};
int i = binsearch(br->restart_count, &restart_key_less, &args);
if (args.error) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}
it->br = br;
if (i > 0) {
i--;
it->next_off = block_reader_restart_offset(br, i);
} else {
it->next_off = br->header_off + 4;
}
/* We're looking for the last entry less/equal than the wanted key, so
we have to go one entry too far and then back up.
*/
while (1) {
block_iter_copy_from(&next, it);
err = block_iter_next(&next, &rec);
if (err < 0)
goto done;
reftable_record_key(&rec, &key);
if (err > 0 || strbuf_cmp(&key, want) >= 0) {
err = 0;
goto done;
}
block_iter_copy_from(it, &next);
}
done:
strbuf_release(&key);
strbuf_release(&next.last_key);
reftable_record_release(&rec);
return err;
}
void block_writer_release(struct block_writer *bw)
{
FREE_AND_NULL(bw->restarts);
strbuf_release(&bw->last_key);
/* the block is not owned. */
}
void reftable_block_done(struct reftable_block *blockp)
{
struct reftable_block_source source = blockp->source;
if (blockp && source.ops)
source.ops->return_block(source.arg, blockp);
blockp->data = NULL;
blockp->len = 0;
blockp->source.ops = NULL;
blockp->source.arg = NULL;
}