Merge branch 'jc/pack-order-tweak'
* jc/pack-order-tweak: pack-objects: optimize "recency order" core: log offset pack data accesses happened
This commit is contained in:
commit
96790ca029
@ -51,6 +51,8 @@ struct object_entry {
|
||||
* objects against.
|
||||
*/
|
||||
unsigned char no_try_delta;
|
||||
unsigned char tagged; /* near the very tip of refs */
|
||||
unsigned char filled; /* assigned write-order */
|
||||
};
|
||||
|
||||
/*
|
||||
@ -96,6 +98,7 @@ static unsigned long window_memory_limit = 0;
|
||||
*/
|
||||
static int *object_ix;
|
||||
static int object_ix_hashsz;
|
||||
static struct object_entry *locate_object_entry(const unsigned char *sha1);
|
||||
|
||||
/*
|
||||
* stats
|
||||
@ -200,6 +203,7 @@ static void copy_pack_data(struct sha1file *f,
|
||||
}
|
||||
}
|
||||
|
||||
/* Return 0 if we will bust the pack-size limit */
|
||||
static unsigned long write_object(struct sha1file *f,
|
||||
struct object_entry *entry,
|
||||
off_t write_offset)
|
||||
@ -434,6 +438,134 @@ static int write_one(struct sha1file *f,
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int mark_tagged(const char *path, const unsigned char *sha1, int flag,
|
||||
void *cb_data)
|
||||
{
|
||||
unsigned char peeled[20];
|
||||
struct object_entry *entry = locate_object_entry(sha1);
|
||||
|
||||
if (entry)
|
||||
entry->tagged = 1;
|
||||
if (!peel_ref(path, peeled)) {
|
||||
entry = locate_object_entry(peeled);
|
||||
if (entry)
|
||||
entry->tagged = 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void add_to_write_order(struct object_entry **wo,
|
||||
int *endp,
|
||||
struct object_entry *e)
|
||||
{
|
||||
if (e->filled)
|
||||
return;
|
||||
wo[(*endp)++] = e;
|
||||
e->filled = 1;
|
||||
}
|
||||
|
||||
static void add_descendants_to_write_order(struct object_entry **wo,
|
||||
int *endp,
|
||||
struct object_entry *e)
|
||||
{
|
||||
struct object_entry *child;
|
||||
|
||||
for (child = e->delta_child; child; child = child->delta_sibling)
|
||||
add_to_write_order(wo, endp, child);
|
||||
for (child = e->delta_child; child; child = child->delta_sibling)
|
||||
add_descendants_to_write_order(wo, endp, child);
|
||||
}
|
||||
|
||||
static void add_family_to_write_order(struct object_entry **wo,
|
||||
int *endp,
|
||||
struct object_entry *e)
|
||||
{
|
||||
struct object_entry *root;
|
||||
|
||||
for (root = e; root->delta; root = root->delta)
|
||||
; /* nothing */
|
||||
add_to_write_order(wo, endp, root);
|
||||
add_descendants_to_write_order(wo, endp, root);
|
||||
}
|
||||
|
||||
static struct object_entry **compute_write_order(void)
|
||||
{
|
||||
int i, wo_end;
|
||||
|
||||
struct object_entry **wo = xmalloc(nr_objects * sizeof(*wo));
|
||||
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
objects[i].tagged = 0;
|
||||
objects[i].filled = 0;
|
||||
objects[i].delta_child = NULL;
|
||||
objects[i].delta_sibling = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fully connect delta_child/delta_sibling network.
|
||||
* Make sure delta_sibling is sorted in the original
|
||||
* recency order.
|
||||
*/
|
||||
for (i = nr_objects - 1; 0 <= i; i--) {
|
||||
struct object_entry *e = &objects[i];
|
||||
if (!e->delta)
|
||||
continue;
|
||||
/* Mark me as the first child */
|
||||
e->delta_sibling = e->delta->delta_child;
|
||||
e->delta->delta_child = e;
|
||||
}
|
||||
|
||||
/*
|
||||
* Mark objects that are at the tip of tags.
|
||||
*/
|
||||
for_each_tag_ref(mark_tagged, NULL);
|
||||
|
||||
/*
|
||||
* Give the commits in the original recency order until
|
||||
* we see a tagged tip.
|
||||
*/
|
||||
for (i = wo_end = 0; i < nr_objects; i++) {
|
||||
if (objects[i].tagged)
|
||||
break;
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Then fill all the tagged tips.
|
||||
*/
|
||||
for (; i < nr_objects; i++) {
|
||||
if (objects[i].tagged)
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* And then all remaining commits and tags.
|
||||
*/
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
if (objects[i].type != OBJ_COMMIT &&
|
||||
objects[i].type != OBJ_TAG)
|
||||
continue;
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* And then all the trees.
|
||||
*/
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
if (objects[i].type != OBJ_TREE)
|
||||
continue;
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Finally all the rest in really tight order
|
||||
*/
|
||||
for (i = 0; i < nr_objects; i++)
|
||||
add_family_to_write_order(wo, &wo_end, &objects[i]);
|
||||
|
||||
return wo;
|
||||
}
|
||||
|
||||
static void write_pack_file(void)
|
||||
{
|
||||
uint32_t i = 0, j;
|
||||
@ -442,10 +574,12 @@ static void write_pack_file(void)
|
||||
struct pack_header hdr;
|
||||
uint32_t nr_remaining = nr_result;
|
||||
time_t last_mtime = 0;
|
||||
struct object_entry **write_order;
|
||||
|
||||
if (progress > pack_to_stdout)
|
||||
progress_state = start_progress("Writing objects", nr_result);
|
||||
written_list = xmalloc(nr_objects * sizeof(*written_list));
|
||||
write_order = compute_write_order();
|
||||
|
||||
do {
|
||||
unsigned char sha1[20];
|
||||
@ -469,7 +603,8 @@ static void write_pack_file(void)
|
||||
offset = sizeof(hdr);
|
||||
nr_written = 0;
|
||||
for (; i < nr_objects; i++) {
|
||||
if (!write_one(f, objects + i, &offset))
|
||||
struct object_entry *e = write_order[i];
|
||||
if (!write_one(f, e, &offset))
|
||||
break;
|
||||
display_progress(progress_state, written);
|
||||
}
|
||||
@ -546,6 +681,7 @@ static void write_pack_file(void)
|
||||
} while (nr_remaining && i < nr_objects);
|
||||
|
||||
free(written_list);
|
||||
free(write_order);
|
||||
stop_progress(&progress_state);
|
||||
if (written != nr_result)
|
||||
die("wrote %"PRIu32" objects while expecting %"PRIu32,
|
||||
|
3
cache.h
3
cache.h
@ -773,6 +773,9 @@ extern int parse_sha1_header(const char *hdr, unsigned long *sizep);
|
||||
/* global flag to enable extra checks when accessing packed objects */
|
||||
extern int do_check_packed_object_crc;
|
||||
|
||||
/* for development: log offset of pack access */
|
||||
extern const char *log_pack_access;
|
||||
|
||||
extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type);
|
||||
|
||||
extern int move_temp_to_file(const char *tmpfile, const char *filename);
|
||||
|
3
config.c
3
config.c
@ -576,6 +576,9 @@ static int git_default_core_config(const char *var, const char *value)
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!strcmp(var, "core.logpackaccess"))
|
||||
return git_config_string(&log_pack_access, var, value);
|
||||
|
||||
if (!strcmp(var, "core.autocrlf")) {
|
||||
if (value && !strcasecmp(value, "input")) {
|
||||
if (core_eol == EOL_CRLF)
|
||||
|
@ -36,6 +36,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
|
||||
size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
|
||||
size_t delta_base_cache_limit = 16 * 1024 * 1024;
|
||||
unsigned long big_file_threshold = 512 * 1024 * 1024;
|
||||
const char *log_pack_access;
|
||||
const char *pager_program;
|
||||
int pager_use_color = 1;
|
||||
const char *editor_program;
|
||||
|
21
sha1_file.c
21
sha1_file.c
@ -1797,6 +1797,24 @@ static void *unpack_delta_entry(struct packed_git *p,
|
||||
return result;
|
||||
}
|
||||
|
||||
static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
|
||||
{
|
||||
static FILE *log_file;
|
||||
|
||||
if (!log_file) {
|
||||
log_file = fopen(log_pack_access, "w");
|
||||
if (!log_file) {
|
||||
error("cannot open pack access log '%s' for writing: %s",
|
||||
log_pack_access, strerror(errno));
|
||||
log_pack_access = NULL;
|
||||
return;
|
||||
}
|
||||
}
|
||||
fprintf(log_file, "%s %"PRIuMAX"\n",
|
||||
p->pack_name, (uintmax_t)obj_offset);
|
||||
fflush(log_file);
|
||||
}
|
||||
|
||||
int do_check_packed_object_crc;
|
||||
|
||||
void *unpack_entry(struct packed_git *p, off_t obj_offset,
|
||||
@ -1806,6 +1824,9 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
|
||||
off_t curpos = obj_offset;
|
||||
void *data;
|
||||
|
||||
if (log_pack_access)
|
||||
write_pack_access_log(p, obj_offset);
|
||||
|
||||
if (do_check_packed_object_crc && p->index_version > 1) {
|
||||
struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
|
||||
unsigned long len = revidx[1].offset - obj_offset;
|
||||
|
Loading…
Reference in New Issue
Block a user