Merge branch 'jk/server-info-rabbit-hole'

Code clean-up around a much-less-important-than-it-used-to-be
update_server_info() funtion.

* jk/server-info-rabbit-hole:
  update_info_refs(): drop unused force parameter
  server-info: drop objdirlen pointer arithmetic
  server-info: drop nr_alloc struct member
  server-info: use strbuf to read old info/packs file
  server-info: simplify cleanup in parse_pack_def()
  server-info: fix blind pointer arithmetic
  http: simplify parsing of remote objects/info/packs
  packfile: fix pack basename computation
  midx: check both pack and index names for containment
  t5319: drop useless --buffer from cat-file
  t5319: fix bogus cat-file argument
  pack-revindex: open index if necessary
  packfile.h: drop extern from function declarations
This commit is contained in:
Junio C Hamano 2019-04-25 16:41:13 +09:00
commit 776f3e1fb7
11 changed files with 173 additions and 111 deletions

35
http.c
View File

@ -2153,11 +2153,11 @@ add_pack:
int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
{
struct http_get_options options = {0};
int ret = 0, i = 0;
char *url, *data;
int ret = 0;
char *url;
const char *data;
struct strbuf buf = STRBUF_INIT;
unsigned char hash[GIT_MAX_RAWSZ];
const unsigned hexsz = the_hash_algo->hexsz;
struct object_id oid;
end_url_with_slash(&buf, base_url);
strbuf_addstr(&buf, "objects/info/packs");
@ -2169,24 +2169,17 @@ int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
goto cleanup;
data = buf.buf;
while (i < buf.len) {
switch (data[i]) {
case 'P':
i++;
if (i + hexsz + 12 <= buf.len &&
starts_with(data + i, " pack-") &&
starts_with(data + i + hexsz + 6, ".pack\n")) {
get_sha1_hex(data + i + 6, hash);
fetch_and_setup_pack_index(packs_head, hash,
base_url);
i += hexsz + 11;
break;
}
default:
while (i < buf.len && data[i] != '\n')
i++;
while (*data) {
if (skip_prefix(data, "P pack-", &data) &&
!parse_oid_hex(data, &oid, &data) &&
skip_prefix(data, ".pack", &data) &&
(*data == '\n' || *data == '\0')) {
fetch_and_setup_pack_index(packs_head, oid.hash, base_url);
} else {
data = strchrnul(data, '\n');
}
i++;
if (*data)
data++; /* skip past newline */
}
cleanup:

36
midx.c
View File

@ -311,7 +311,39 @@ int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct mu
return nth_midxed_pack_entry(m, e, pos);
}
int midx_contains_pack(struct multi_pack_index *m, const char *idx_name)
/* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
const char *idx_name)
{
/* Skip past any initial matching prefix. */
while (*idx_name && *idx_name == *idx_or_pack_name) {
idx_name++;
idx_or_pack_name++;
}
/*
* If we didn't match completely, we may have matched "pack-1234." and
* be left with "idx" and "pack" respectively, which is also OK. We do
* not have to check for "idx" and "idx", because that would have been
* a complete match (and in that case these strcmps will be false, but
* we'll correctly return 0 from the final strcmp() below.
*
* Technically this matches "fooidx" and "foopack", but we'd never have
* such names in the first place.
*/
if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
return 0;
/*
* This not only checks for a complete match, but also orders based on
* the first non-identical character, which means our ordering will
* match a raw strcmp(). That makes it OK to use this to binary search
* a naively-sorted list.
*/
return strcmp(idx_or_pack_name, idx_name);
}
int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
{
uint32_t first = 0, last = m->num_packs;
@ -321,7 +353,7 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_name)
int cmp;
current = m->pack_names[mid];
cmp = strcmp(idx_name, current);
cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
if (!cmp)
return 1;
if (cmp > 0) {

2
midx.h
View File

@ -43,7 +43,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid,
struct multi_pack_index *m,
uint32_t n);
int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m);
int midx_contains_pack(struct multi_pack_index *m, const char *idx_name);
int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name);
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local);
int write_midx_file(const char *object_dir);

View File

@ -308,7 +308,8 @@ static int load_pack_bitmap(struct bitmap_index *bitmap_git)
bitmap_git->bitmaps = kh_init_sha1();
bitmap_git->ext_index.positions = kh_init_sha1_pos();
load_pack_revindex(bitmap_git->pack);
if (load_pack_revindex(bitmap_git->pack))
goto failed;
if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) ||
!(bitmap_git->trees = read_bitmap_1(bitmap_git)) ||

View File

@ -1,6 +1,7 @@
#include "cache.h"
#include "pack-revindex.h"
#include "object-store.h"
#include "packfile.h"
/*
* Pack index for existing packs give us easy access to the offsets into
@ -158,10 +159,14 @@ static void create_pack_revindex(struct packed_git *p)
sort_revindex(p->revindex, num_ent, p->pack_size);
}
void load_pack_revindex(struct packed_git *p)
int load_pack_revindex(struct packed_git *p)
{
if (!p->revindex)
if (!p->revindex) {
if (open_pack_index(p))
return -1;
create_pack_revindex(p);
}
return 0;
}
int find_revindex_position(struct packed_git *p, off_t ofs)
@ -188,7 +193,9 @@ struct revindex_entry *find_pack_revindex(struct packed_git *p, off_t ofs)
{
int pos;
load_pack_revindex(p);
if (load_pack_revindex(p))
return NULL;
pos = find_revindex_position(p, ofs);
if (pos < 0)

View File

@ -8,7 +8,7 @@ struct revindex_entry {
unsigned int nr;
};
void load_pack_revindex(struct packed_git *p);
int load_pack_revindex(struct packed_git *p);
int find_revindex_position(struct packed_git *p, off_t ofs);
struct revindex_entry *find_pack_revindex(struct packed_git *p, off_t ofs);

View File

@ -466,6 +466,16 @@ static unsigned int get_max_fd_limit(void)
#endif
}
const char *pack_basename(struct packed_git *p)
{
const char *ret = strrchr(p->pack_name, '/');
if (ret)
ret = ret + 1; /* skip past slash */
else
ret = p->pack_name; /* we only have a base */
return ret;
}
/*
* Do not call this directly as this leaks p->pack_fd on error return;
* call open_packed_git() instead.
@ -482,7 +492,7 @@ static int open_packed_git_1(struct packed_git *p)
if (!p->index_data) {
struct multi_pack_index *m;
const char *pack_name = strrchr(p->pack_name, '/');
const char *pack_name = pack_basename(p);
for (m = the_repository->objects->multi_pack_index;
m; m = m->next) {
@ -2023,8 +2033,10 @@ int for_each_object_in_pack(struct packed_git *p,
uint32_t i;
int r = 0;
if (flags & FOR_EACH_OBJECT_PACK_ORDER)
load_pack_revindex(p);
if (flags & FOR_EACH_OBJECT_PACK_ORDER) {
if (load_pack_revindex(p))
return -1;
}
for (i = 0; i < p->num_objects; i++) {
uint32_t pos;

View File

@ -15,23 +15,29 @@ struct object_info;
*
* Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx"
*/
extern char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext);
char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext);
/*
* Return the name of the (local) packfile with the specified sha1 in
* its name. The return value is a pointer to memory that is
* overwritten each time this function is called.
*/
extern char *sha1_pack_name(const unsigned char *sha1);
char *sha1_pack_name(const unsigned char *sha1);
/*
* Return the name of the (local) pack index file with the specified
* sha1 in its name. The return value is a pointer to memory that is
* overwritten each time this function is called.
*/
extern char *sha1_pack_index_name(const unsigned char *sha1);
char *sha1_pack_index_name(const unsigned char *sha1);
extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
/*
* Return the basename of the packfile, omitting any containing directory
* (e.g., "pack-1234abcd[...].pack").
*/
const char *pack_basename(struct packed_git *p);
struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len,
const char *file_pach, void *data);
@ -45,8 +51,8 @@ void for_each_file_in_pack_dir(const char *objdir,
#define PACKDIR_FILE_GARBAGE 4
extern void (*report_garbage)(unsigned seen_bits, const char *path);
extern void reprepare_packed_git(struct repository *r);
extern void install_packed_git(struct repository *r, struct packed_git *pack);
void reprepare_packed_git(struct repository *r);
void install_packed_git(struct repository *r, struct packed_git *pack);
struct packed_git *get_packed_git(struct repository *r);
struct list_head *get_packed_git_mru(struct repository *r);
@ -59,34 +65,34 @@ struct packed_git *get_all_packs(struct repository *r);
*/
unsigned long approximate_object_count(void);
extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
struct packed_git *packs);
struct packed_git *find_sha1_pack(const unsigned char *sha1,
struct packed_git *packs);
extern void pack_report(void);
void pack_report(void);
/*
* mmap the index file for the specified packfile (if it is not
* already mmapped). Return 0 on success.
*/
extern int open_pack_index(struct packed_git *);
int open_pack_index(struct packed_git *);
/*
* munmap the index file for the specified packfile (if it is
* currently mmapped).
*/
extern void close_pack_index(struct packed_git *);
void close_pack_index(struct packed_git *);
int close_pack_fd(struct packed_git *p);
extern uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
extern void close_pack_windows(struct packed_git *);
extern void close_pack(struct packed_git *);
extern void close_all_packs(struct raw_object_store *o);
extern void unuse_pack(struct pack_window **);
extern void clear_delta_base_cache(void);
extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local);
unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
void close_pack_windows(struct packed_git *);
void close_pack(struct packed_git *);
void close_all_packs(struct raw_object_store *o);
void unuse_pack(struct pack_window **);
void clear_delta_base_cache(void);
struct packed_git *add_packed_git(const char *path, size_t path_len, int local);
/*
* Make sure that a pointer access into an mmap'd index file is within bounds,
@ -96,7 +102,7 @@ extern struct packed_git *add_packed_git(const char *path, size_t path_len, int
* (like the 64-bit extended offset table), as we compare the size to the
* fixed-length parts when we open the file.
*/
extern void check_pack_index_ptr(const struct packed_git *p, const void *ptr);
void check_pack_index_ptr(const struct packed_git *p, const void *ptr);
/*
* Perform binary search on a pack-index for a given oid. Packfile is expected to
@ -112,59 +118,59 @@ int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32
* at the SHA-1 within the mmapped index. Return NULL if there is an
* error.
*/
extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n);
const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n);
/*
* Like nth_packed_object_sha1, but write the data into the object specified by
* the the first argument. Returns the first argument on success, and NULL on
* error.
*/
extern const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n);
const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n);
/*
* Return the offset of the nth object within the specified packfile.
* The index must already be opened.
*/
extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t n);
off_t nth_packed_object_offset(const struct packed_git *, uint32_t n);
/*
* If the object named sha1 is present in the specified packfile,
* return its offset within the packfile; otherwise, return 0.
*/
extern off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *);
off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *);
extern int is_pack_valid(struct packed_git *);
extern void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *);
extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
int is_pack_valid(struct packed_git *);
void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *);
unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
extern void release_pack_memory(size_t);
void release_pack_memory(size_t);
/* global flag to enable extra checks when accessing packed objects */
extern int do_check_packed_object_crc;
extern int packed_object_info(struct repository *r,
struct packed_git *pack,
off_t offset, struct object_info *);
int packed_object_info(struct repository *r,
struct packed_git *pack,
off_t offset, struct object_info *);
extern void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1);
extern const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1);
void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1);
const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1);
/*
* Iff a pack file in the given repository contains the object named by sha1,
* return true and store its location to e.
*/
extern int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e);
int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e);
extern int has_object_pack(const struct object_id *oid);
int has_object_pack(const struct object_id *oid);
extern int has_pack_index(const unsigned char *sha1);
int has_pack_index(const unsigned char *sha1);
/*
* Return 1 if an object in a promisor packfile is or refers to the given
* object, 0 otherwise.
*/
extern int is_promisor_object(const struct object_id *oid);
int is_promisor_object(const struct object_id *oid);
/*
* Expose a function for fuzz testing.
@ -176,7 +182,7 @@ extern int is_promisor_object(const struct object_id *oid);
* have a convenient entry-point for fuzz testing. For real uses, you should
* probably use open_pack_index() or parse_pack_index() instead.
*/
extern int load_idx(const char *path, const unsigned int hashsz, void *idx_map,
size_t idx_size, struct packed_git *p);
int load_idx(const char *path, const unsigned int hashsz, void *idx_map,
size_t idx_size, struct packed_git *p);
#endif

View File

@ -78,7 +78,7 @@ static int generate_info_refs(FILE *fp)
return for_each_ref(add_info_ref, fp);
}
static int update_info_refs(int force)
static int update_info_refs(void)
{
char *path = git_pathdup("info/refs");
int ret = update_info_file(path, generate_info_refs);
@ -91,19 +91,15 @@ static struct pack_info {
struct packed_git *p;
int old_num;
int new_num;
int nr_alloc;
} **info;
static int num_pack;
static const char *objdir;
static int objdirlen;
static struct pack_info *find_pack_by_name(const char *name)
{
int i;
for (i = 0; i < num_pack; i++) {
struct packed_git *p = info[i]->p;
/* skip "/pack/" after ".git/objects" */
if (!strcmp(p->pack_name + objdirlen + 6, name))
if (!strcmp(pack_basename(p), name))
return info[i];
}
return NULL;
@ -112,9 +108,9 @@ static struct pack_info *find_pack_by_name(const char *name)
/* Returns non-zero when we detect that the info in the
* old file is useless.
*/
static int parse_pack_def(const char *line, int old_cnt)
static int parse_pack_def(const char *packname, int old_cnt)
{
struct pack_info *i = find_pack_by_name(line + 2);
struct pack_info *i = find_pack_by_name(packname);
if (i) {
i->old_num = old_cnt;
return 0;
@ -131,39 +127,40 @@ static int parse_pack_def(const char *line, int old_cnt)
static int read_pack_info_file(const char *infofile)
{
FILE *fp;
char line[1000];
struct strbuf line = STRBUF_INIT;
int old_cnt = 0;
int stale = 1;
fp = fopen_or_warn(infofile, "r");
if (!fp)
return 1; /* nonexistent is not an error. */
while (fgets(line, sizeof(line), fp)) {
int len = strlen(line);
if (len && line[len-1] == '\n')
line[--len] = 0;
while (strbuf_getline(&line, fp) != EOF) {
const char *arg;
if (!len)
if (!line.len)
continue;
switch (line[0]) {
case 'P': /* P name */
if (parse_pack_def(line, old_cnt++))
if (skip_prefix(line.buf, "P ", &arg)) {
/* P name */
if (parse_pack_def(arg, old_cnt++))
goto out_stale;
break;
case 'D': /* we used to emit D but that was misguided. */
case 'T': /* we used to emit T but nobody uses it. */
} else if (line.buf[0] == 'D') {
/* we used to emit D but that was misguided. */
goto out_stale;
default:
error("unrecognized: %s", line);
break;
} else if (line.buf[0] == 'T') {
/* we used to emit T but nobody uses it. */
goto out_stale;
} else {
error("unrecognized: %s", line.buf);
}
}
fclose(fp);
return 0;
stale = 0;
out_stale:
strbuf_release(&line);
fclose(fp);
return 1;
return stale;
}
static int compare_info(const void *a_, const void *b_)
@ -196,9 +193,6 @@ static void init_pack_info(const char *infofile, int force)
int stale;
int i = 0;
objdir = get_object_directory();
objdirlen = strlen(objdir);
for (p = get_all_packs(the_repository); p; p = p->next) {
/* we ignore things on alternate path since they are
* not available to the pullers in general.
@ -212,6 +206,7 @@ static void init_pack_info(const char *infofile, int force)
for (i = 0, p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local)
continue;
assert(i < num_pack);
info[i] = xcalloc(1, sizeof(struct pack_info));
info[i]->p = p;
info[i]->old_num = -1;
@ -245,7 +240,7 @@ static int write_pack_info_file(FILE *fp)
{
int i;
for (i = 0; i < num_pack; i++) {
if (fprintf(fp, "P %s\n", info[i]->p->pack_name + objdirlen + 6) < 0)
if (fprintf(fp, "P %s\n", pack_basename(info[i]->p)) < 0)
return -1;
}
if (fputc('\n', fp) == EOF)
@ -274,7 +269,7 @@ int update_server_info(int force)
*/
int errs = 0;
errs = errs | update_info_refs(force);
errs = errs | update_info_refs();
errs = errs | update_info_packs(force);
/* remove leftover rev-cache file if there is any */

View File

@ -86,13 +86,14 @@ test_expect_success 'write midx with one v1 pack' '
'
midx_git_two_modes () {
git -c core.multiPackIndex=false $1 >expect &&
git -c core.multiPackIndex=true $1 >actual &&
if [ "$2" = "sorted" ]
then
git -c core.multiPackIndex=false $1 | sort >expect &&
git -c core.multiPackIndex=true $1 | sort >actual
else
git -c core.multiPackIndex=false $1 >expect &&
git -c core.multiPackIndex=true $1 >actual
sort <expect >expect.sorted &&
mv expect.sorted expect &&
sort <actual >actual.sorted &&
mv actual.sorted actual
fi &&
test_cmp expect actual
}
@ -103,8 +104,8 @@ compare_results_with_midx () {
midx_git_two_modes "rev-list --objects --all" &&
midx_git_two_modes "log --raw" &&
midx_git_two_modes "count-objects --verbose" &&
midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check" &&
midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check --unsorted" sorted
midx_git_two_modes "cat-file --batch-all-objects --batch-check" &&
midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted
'
}
@ -116,6 +117,20 @@ test_expect_success 'write midx with one v2 pack' '
compare_results_with_midx "one v2 pack"
test_expect_success 'corrupt idx not opened' '
idx=$(test-tool read-midx $objdir | grep "\.idx\$") &&
mv $objdir/pack/$idx backup-$idx &&
test_when_finished "mv backup-\$idx \$objdir/pack/\$idx" &&
# This is the minimum size for a sha-1 based .idx; this lets
# us pass perfunctory tests, but anything that actually opens and reads
# the idx file will complain.
test_copy_bytes 1064 <backup-$idx >$objdir/pack/$idx &&
git -c core.multiPackIndex=true rev-list --objects --all 2>err &&
test_must_be_empty err
'
test_expect_success 'add more objects' '
for i in $(test_seq 6 10)
do

View File

@ -90,6 +90,7 @@ test_expect_success 'fetch notices corrupt pack' '
test_expect_success 'fetch notices corrupt idx' '
cp -R "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_pack.git "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
(cd "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
rm -f objects/pack/multi-pack-index &&
p=$(ls objects/pack/pack-*.idx) &&
chmod u+w $p &&
printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc