Merge branch 'ab/commit-graph-write-progress'
The codepath to show progress meter while writing out commit-graph file has been improved. * ab/commit-graph-write-progress: commit-graph write: emit a percentage for all progress commit-graph write: add itermediate progress commit-graph write: remove empty line for readability commit-graph write: add more descriptive progress output commit-graph write: show progress for object search commit-graph write: more descriptive "writing out" output commit-graph write: add "Writing out" progress output commit-graph: don't call write_graph_chunk_extra_edges() unnecessarily commit-graph: rename "large edges" to "extra edges"
This commit is contained in:
Junio C Hamano
commit
e5eac57356
@ -76,7 +76,7 @@ CHUNK DATA:
|
||||
of the ith commit. Stores value 0x7000000 if no parent in that
|
||||
position. If there are more than two parents, the second value
|
||||
has its most-significant bit on and the other bits store an array
|
||||
position into the Large Edge List chunk.
|
||||
position into the Extra Edge List chunk.
|
||||
* The next 8 bytes store the generation number of the commit and
|
||||
the commit time in seconds since EPOCH. The generation number
|
||||
uses the higher 30 bits of the first 4 bytes, while the commit
|
||||
@ -84,7 +84,7 @@ CHUNK DATA:
|
||||
2 bits of the lowest byte, storing the 33rd and 34th bit of the
|
||||
commit time.
|
||||
|
||||
Large Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional]
|
||||
Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional]
|
||||
This list of 4-byte values store the second through nth parents for
|
||||
all octopus merges. The second parent value in the commit data stores
|
||||
an array position within this list along with the most-significant bit
|
||||
|
||||
@ -110,8 +110,8 @@ static int graph_read(int argc, const char **argv)
|
||||
printf(" oid_lookup");
|
||||
if (graph->chunk_commit_data)
|
||||
printf(" commit_metadata");
|
||||
if (graph->chunk_large_edges)
|
||||
printf(" large_edges");
|
||||
if (graph->chunk_extra_edges)
|
||||
printf(" extra_edges");
|
||||
printf("\n");
|
||||
|
||||
UNLEAK(graph);
|
||||
|
||||
130
commit-graph.c
130
commit-graph.c
@ -21,14 +21,14 @@
|
||||
#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
|
||||
#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
|
||||
#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
|
||||
#define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
|
||||
#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
|
||||
|
||||
#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
|
||||
|
||||
#define GRAPH_VERSION_1 0x1
|
||||
#define GRAPH_VERSION GRAPH_VERSION_1
|
||||
|
||||
#define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
|
||||
#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
|
||||
#define GRAPH_EDGE_LAST_MASK 0x7fffffff
|
||||
#define GRAPH_PARENT_NONE 0x70000000
|
||||
|
||||
@ -209,11 +209,11 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
|
||||
graph->chunk_commit_data = data + chunk_offset;
|
||||
break;
|
||||
|
||||
case GRAPH_CHUNKID_LARGEEDGES:
|
||||
if (graph->chunk_large_edges)
|
||||
case GRAPH_CHUNKID_EXTRAEDGES:
|
||||
if (graph->chunk_extra_edges)
|
||||
chunk_repeated = 1;
|
||||
else
|
||||
graph->chunk_large_edges = data + chunk_offset;
|
||||
graph->chunk_extra_edges = data + chunk_offset;
|
||||
break;
|
||||
}
|
||||
|
||||
@ -374,12 +374,12 @@ static int fill_commit_in_graph(struct repository *r,
|
||||
edge_value = get_be32(commit_data + g->hash_len + 4);
|
||||
if (edge_value == GRAPH_PARENT_NONE)
|
||||
return 1;
|
||||
if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
|
||||
if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
|
||||
pptr = insert_parent_or_die(r, g, edge_value, pptr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
|
||||
parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
|
||||
4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
|
||||
do {
|
||||
edge_value = get_be32(parent_data_ptr);
|
||||
@ -466,7 +466,9 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit
|
||||
|
||||
static void write_graph_chunk_fanout(struct hashfile *f,
|
||||
struct commit **commits,
|
||||
int nr_commits)
|
||||
int nr_commits,
|
||||
struct progress *progress,
|
||||
uint64_t *progress_cnt)
|
||||
{
|
||||
int i, count = 0;
|
||||
struct commit **list = commits;
|
||||
@ -480,6 +482,7 @@ static void write_graph_chunk_fanout(struct hashfile *f,
|
||||
while (count < nr_commits) {
|
||||
if ((*list)->object.oid.hash[0] != i)
|
||||
break;
|
||||
display_progress(progress, ++*progress_cnt);
|
||||
count++;
|
||||
list++;
|
||||
}
|
||||
@ -489,12 +492,16 @@ static void write_graph_chunk_fanout(struct hashfile *f,
|
||||
}
|
||||
|
||||
static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
|
||||
struct commit **commits, int nr_commits)
|
||||
struct commit **commits, int nr_commits,
|
||||
struct progress *progress,
|
||||
uint64_t *progress_cnt)
|
||||
{
|
||||
struct commit **list = commits;
|
||||
int count;
|
||||
for (count = 0; count < nr_commits; count++, list++)
|
||||
for (count = 0; count < nr_commits; count++, list++) {
|
||||
display_progress(progress, ++*progress_cnt);
|
||||
hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
|
||||
}
|
||||
}
|
||||
|
||||
static const unsigned char *commit_to_sha1(size_t index, void *table)
|
||||
@ -504,7 +511,9 @@ static const unsigned char *commit_to_sha1(size_t index, void *table)
|
||||
}
|
||||
|
||||
static void write_graph_chunk_data(struct hashfile *f, int hash_len,
|
||||
struct commit **commits, int nr_commits)
|
||||
struct commit **commits, int nr_commits,
|
||||
struct progress *progress,
|
||||
uint64_t *progress_cnt)
|
||||
{
|
||||
struct commit **list = commits;
|
||||
struct commit **last = commits + nr_commits;
|
||||
@ -514,6 +523,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
|
||||
struct commit_list *parent;
|
||||
int edge_value;
|
||||
uint32_t packedDate[2];
|
||||
display_progress(progress, ++*progress_cnt);
|
||||
|
||||
parse_commit(*list);
|
||||
hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
|
||||
@ -542,7 +552,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
|
||||
if (!parent)
|
||||
edge_value = GRAPH_PARENT_NONE;
|
||||
else if (parent->next)
|
||||
edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
|
||||
edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
|
||||
else {
|
||||
edge_value = sha1_pos(parent->item->object.oid.hash,
|
||||
commits,
|
||||
@ -556,7 +566,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
|
||||
|
||||
hashwrite_be32(f, edge_value);
|
||||
|
||||
if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
|
||||
if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
|
||||
do {
|
||||
num_extra_edges++;
|
||||
parent = parent->next;
|
||||
@ -577,9 +587,11 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
|
||||
}
|
||||
}
|
||||
|
||||
static void write_graph_chunk_large_edges(struct hashfile *f,
|
||||
static void write_graph_chunk_extra_edges(struct hashfile *f,
|
||||
struct commit **commits,
|
||||
int nr_commits)
|
||||
int nr_commits,
|
||||
struct progress *progress,
|
||||
uint64_t *progress_cnt)
|
||||
{
|
||||
struct commit **list = commits;
|
||||
struct commit **last = commits + nr_commits;
|
||||
@ -587,6 +599,9 @@ static void write_graph_chunk_large_edges(struct hashfile *f,
|
||||
|
||||
while (list < last) {
|
||||
int num_parents = 0;
|
||||
|
||||
display_progress(progress, ++*progress_cnt);
|
||||
|
||||
for (parent = (*list)->parents; num_parents < 3 && parent;
|
||||
parent = parent->next)
|
||||
num_parents++;
|
||||
@ -680,15 +695,15 @@ static void add_missing_parents(struct packed_oid_list *oids, struct commit *com
|
||||
|
||||
static void close_reachable(struct packed_oid_list *oids, int report_progress)
|
||||
{
|
||||
int i, j;
|
||||
int i;
|
||||
struct commit *commit;
|
||||
struct progress *progress = NULL;
|
||||
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Loading known commits in commit graph"), j = 0);
|
||||
_("Loading known commits in commit graph"), oids->nr);
|
||||
for (i = 0; i < oids->nr; i++) {
|
||||
display_progress(progress, ++j);
|
||||
display_progress(progress, i + 1);
|
||||
commit = lookup_commit(the_repository, &oids->list[i]);
|
||||
if (commit)
|
||||
commit->object.flags |= UNINTERESTING;
|
||||
@ -702,9 +717,9 @@ static void close_reachable(struct packed_oid_list *oids, int report_progress)
|
||||
*/
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Expanding reachable commits in commit graph"), j = 0);
|
||||
_("Expanding reachable commits in commit graph"), oids->nr);
|
||||
for (i = 0; i < oids->nr; i++) {
|
||||
display_progress(progress, ++j);
|
||||
display_progress(progress, i + 1);
|
||||
commit = lookup_commit(the_repository, &oids->list[i]);
|
||||
|
||||
if (commit && !parse_commit(commit))
|
||||
@ -714,9 +729,9 @@ static void close_reachable(struct packed_oid_list *oids, int report_progress)
|
||||
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Clearing commit marks in commit graph"), j = 0);
|
||||
_("Clearing commit marks in commit graph"), oids->nr);
|
||||
for (i = 0; i < oids->nr; i++) {
|
||||
display_progress(progress, ++j);
|
||||
display_progress(progress, i + 1);
|
||||
commit = lookup_commit(the_repository, &oids->list[i]);
|
||||
|
||||
if (commit)
|
||||
@ -811,12 +826,16 @@ void write_commit_graph(const char *obj_dir,
|
||||
struct commit_list *parent;
|
||||
struct progress *progress = NULL;
|
||||
const unsigned hashsz = the_hash_algo->rawsz;
|
||||
uint64_t progress_cnt = 0;
|
||||
struct strbuf progress_title = STRBUF_INIT;
|
||||
unsigned long approx_nr_objects;
|
||||
|
||||
if (!commit_graph_compatible(the_repository))
|
||||
return;
|
||||
|
||||
oids.nr = 0;
|
||||
oids.alloc = approximate_object_count() / 32;
|
||||
approx_nr_objects = approximate_object_count();
|
||||
oids.alloc = approx_nr_objects / 32;
|
||||
oids.progress = NULL;
|
||||
oids.progress_done = 0;
|
||||
|
||||
@ -846,8 +865,12 @@ void write_commit_graph(const char *obj_dir,
|
||||
strbuf_addf(&packname, "%s/pack/", obj_dir);
|
||||
dirlen = packname.len;
|
||||
if (report_progress) {
|
||||
oids.progress = start_delayed_progress(
|
||||
_("Finding commits for commit graph"), 0);
|
||||
strbuf_addf(&progress_title,
|
||||
Q_("Finding commits for commit graph in %d pack",
|
||||
"Finding commits for commit graph in %d packs",
|
||||
pack_indexes->nr),
|
||||
pack_indexes->nr);
|
||||
oids.progress = start_delayed_progress(progress_title.buf, 0);
|
||||
oids.progress_done = 0;
|
||||
}
|
||||
for (i = 0; i < pack_indexes->nr; i++) {
|
||||
@ -865,14 +888,20 @@ void write_commit_graph(const char *obj_dir,
|
||||
free(p);
|
||||
}
|
||||
stop_progress(&oids.progress);
|
||||
strbuf_reset(&progress_title);
|
||||
strbuf_release(&packname);
|
||||
}
|
||||
|
||||
if (commit_hex) {
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Finding commits for commit graph"),
|
||||
commit_hex->nr);
|
||||
if (report_progress) {
|
||||
strbuf_addf(&progress_title,
|
||||
Q_("Finding commits for commit graph from %d ref",
|
||||
"Finding commits for commit graph from %d refs",
|
||||
commit_hex->nr),
|
||||
commit_hex->nr);
|
||||
progress = start_delayed_progress(progress_title.buf,
|
||||
commit_hex->nr);
|
||||
}
|
||||
for (i = 0; i < commit_hex->nr; i++) {
|
||||
const char *end;
|
||||
struct object_id oid;
|
||||
@ -892,26 +921,36 @@ void write_commit_graph(const char *obj_dir,
|
||||
}
|
||||
}
|
||||
stop_progress(&progress);
|
||||
strbuf_reset(&progress_title);
|
||||
}
|
||||
|
||||
if (!pack_indexes && !commit_hex) {
|
||||
if (report_progress)
|
||||
oids.progress = start_delayed_progress(
|
||||
_("Finding commits for commit graph"), 0);
|
||||
_("Finding commits for commit graph among packed objects"),
|
||||
approx_nr_objects);
|
||||
for_each_packed_object(add_packed_commits, &oids,
|
||||
FOR_EACH_OBJECT_PACK_ORDER);
|
||||
if (oids.progress_done < approx_nr_objects)
|
||||
display_progress(oids.progress, approx_nr_objects);
|
||||
stop_progress(&oids.progress);
|
||||
}
|
||||
|
||||
close_reachable(&oids, report_progress);
|
||||
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Counting distinct commits in commit graph"),
|
||||
oids.nr);
|
||||
display_progress(progress, 0); /* TODO: Measure QSORT() progress */
|
||||
QSORT(oids.list, oids.nr, commit_compare);
|
||||
|
||||
count_distinct = 1;
|
||||
for (i = 1; i < oids.nr; i++) {
|
||||
display_progress(progress, i + 1);
|
||||
if (!oideq(&oids.list[i - 1], &oids.list[i]))
|
||||
count_distinct++;
|
||||
}
|
||||
stop_progress(&progress);
|
||||
|
||||
if (count_distinct >= GRAPH_EDGE_LAST_MASK)
|
||||
die(_("the commit graph format cannot write %d commits"), count_distinct);
|
||||
@ -921,8 +960,13 @@ void write_commit_graph(const char *obj_dir,
|
||||
ALLOC_ARRAY(commits.list, commits.alloc);
|
||||
|
||||
num_extra_edges = 0;
|
||||
if (report_progress)
|
||||
progress = start_delayed_progress(
|
||||
_("Finding extra edges in commit graph"),
|
||||
oids.nr);
|
||||
for (i = 0; i < oids.nr; i++) {
|
||||
int num_parents = 0;
|
||||
display_progress(progress, i + 1);
|
||||
if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
|
||||
continue;
|
||||
|
||||
@ -939,6 +983,7 @@ void write_commit_graph(const char *obj_dir,
|
||||
commits.nr++;
|
||||
}
|
||||
num_chunks = num_extra_edges ? 4 : 3;
|
||||
stop_progress(&progress);
|
||||
|
||||
if (commits.nr >= GRAPH_EDGE_LAST_MASK)
|
||||
die(_("too many commits to write graph"));
|
||||
@ -966,7 +1011,7 @@ void write_commit_graph(const char *obj_dir,
|
||||
chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
|
||||
chunk_ids[2] = GRAPH_CHUNKID_DATA;
|
||||
if (num_extra_edges)
|
||||
chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
|
||||
chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
|
||||
else
|
||||
chunk_ids[3] = 0;
|
||||
chunk_ids[4] = 0;
|
||||
@ -986,10 +1031,23 @@ void write_commit_graph(const char *obj_dir,
|
||||
hashwrite(f, chunk_write, 12);
|
||||
}
|
||||
|
||||
write_graph_chunk_fanout(f, commits.list, commits.nr);
|
||||
write_graph_chunk_oids(f, hashsz, commits.list, commits.nr);
|
||||
write_graph_chunk_data(f, hashsz, commits.list, commits.nr);
|
||||
write_graph_chunk_large_edges(f, commits.list, commits.nr);
|
||||
if (report_progress) {
|
||||
strbuf_addf(&progress_title,
|
||||
Q_("Writing out commit graph in %d pass",
|
||||
"Writing out commit graph in %d passes",
|
||||
num_chunks),
|
||||
num_chunks);
|
||||
progress = start_delayed_progress(
|
||||
progress_title.buf,
|
||||
num_chunks * commits.nr);
|
||||
}
|
||||
write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
|
||||
write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
|
||||
write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
|
||||
if (num_extra_edges)
|
||||
write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
|
||||
stop_progress(&progress);
|
||||
strbuf_release(&progress_title);
|
||||
|
||||
close_commit_graph(the_repository);
|
||||
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
|
||||
|
||||
@ -49,7 +49,7 @@ struct commit_graph {
|
||||
const uint32_t *chunk_oid_fanout;
|
||||
const unsigned char *chunk_oid_lookup;
|
||||
const unsigned char *chunk_commit_data;
|
||||
const unsigned char *chunk_large_edges;
|
||||
const unsigned char *chunk_extra_edges;
|
||||
};
|
||||
|
||||
struct commit_graph *load_commit_graph_one(const char *graph_file);
|
||||
|
||||
@ -122,7 +122,7 @@ test_expect_success 'write graph with merges' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
git commit-graph write &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "10" "large_edges"
|
||||
graph_read_expect "10" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'merge 1 vs 2' full merge/1 merge/2
|
||||
@ -157,7 +157,7 @@ test_expect_success 'write graph with new commit' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
git commit-graph write &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "11" "large_edges"
|
||||
graph_read_expect "11" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'full graph, commit 8 vs merge 1' full commits/8 merge/1
|
||||
@ -167,7 +167,7 @@ test_expect_success 'write graph with nothing new' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
git commit-graph write &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "11" "large_edges"
|
||||
graph_read_expect "11" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'cleared graph, commit 8 vs merge 1' full commits/8 merge/1
|
||||
@ -177,7 +177,7 @@ test_expect_success 'build graph from latest pack with closure' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
cat new-idx | git commit-graph write --stdin-packs &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "9" "large_edges"
|
||||
graph_read_expect "9" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'graph from pack, commit 8 vs merge 1' full commits/8 merge/1
|
||||
@ -200,7 +200,7 @@ test_expect_success 'build graph from commits with append' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
git rev-parse merge/3 | git commit-graph write --stdin-commits --append &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "10" "large_edges"
|
||||
graph_read_expect "10" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
|
||||
@ -210,7 +210,7 @@ test_expect_success 'build graph using --reachable' '
|
||||
cd "$TRASH_DIRECTORY/full" &&
|
||||
git commit-graph write --reachable &&
|
||||
test_path_is_file $objdir/info/commit-graph &&
|
||||
graph_read_expect "11" "large_edges"
|
||||
graph_read_expect "11" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
|
||||
@ -231,7 +231,7 @@ test_expect_success 'write graph in bare repo' '
|
||||
cd "$TRASH_DIRECTORY/bare" &&
|
||||
git commit-graph write &&
|
||||
test_path_is_file $baredir/info/commit-graph &&
|
||||
graph_read_expect "11" "large_edges"
|
||||
graph_read_expect "11" "extra_edges"
|
||||
'
|
||||
|
||||
graph_git_behavior 'bare repo with graph, commit 8 vs merge 1' bare commits/8 merge/1
|
||||
|
||||
Loading…
Reference in New Issue
Block a user