Merge branch 'sp/maint-dumb-http-pack-reidx' into maint
* sp/maint-dumb-http-pack-reidx: http.c::new_http_pack_request: do away with the temp variable filename http-fetch: Use temporary files for pack-*.idx until verified http-fetch: Use index-pack rather than verify-pack to check packs Allow parse_pack_index on temporary files Extract verify_pack_index for reuse from verify_pack Introduce close_pack_index to permit replacement http.c: Remove unnecessary strdup of sha1_to_hex result http.c: Don't store destination name in request structures http.c: Drop useless != NULL test in finish_http_pack_request http.c: Tiny refactoring of finish_http_pack_request t5550-http-fetch: Use subshell for repository operations http.c: Remove bad free of static block
This commit is contained in:
Junio C Hamano
commit
161cbf0b8e
3
cache.h
3
cache.h
@ -905,7 +905,7 @@ struct extra_have_objects {
|
||||
extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags, struct extra_have_objects *);
|
||||
extern int server_supports(const char *feature);
|
||||
|
||||
extern struct packed_git *parse_pack_index(unsigned char *sha1);
|
||||
extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
|
||||
|
||||
extern void prepare_packed_git(void);
|
||||
extern void reprepare_packed_git(void);
|
||||
@ -916,6 +916,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
|
||||
|
||||
extern void pack_report(void);
|
||||
extern int open_pack_index(struct packed_git *);
|
||||
extern void close_pack_index(struct packed_git *);
|
||||
extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
|
||||
extern void close_pack_windows(struct packed_git *);
|
||||
extern void unuse_pack(struct pack_window **);
|
||||
|
||||
@ -510,7 +510,7 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c
|
||||
ret = error("File %s has bad hash", hex);
|
||||
} else if (req->rename < 0) {
|
||||
ret = error("unable to write sha1 filename %s",
|
||||
req->filename);
|
||||
sha1_file_name(req->sha1));
|
||||
}
|
||||
|
||||
release_http_object_request(req);
|
||||
|
||||
137
http.c
137
http.c
@ -1,6 +1,7 @@
|
||||
#include "http.h"
|
||||
#include "pack.h"
|
||||
#include "sideband.h"
|
||||
#include "run-command.h"
|
||||
|
||||
int data_received;
|
||||
int active_requests;
|
||||
@ -896,47 +897,67 @@ int http_fetch_ref(const char *base, struct ref *ref)
|
||||
}
|
||||
|
||||
/* Helpers for fetching packs */
|
||||
static int fetch_pack_index(unsigned char *sha1, const char *base_url)
|
||||
static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
|
||||
{
|
||||
int ret = 0;
|
||||
char *hex = xstrdup(sha1_to_hex(sha1));
|
||||
char *filename;
|
||||
char *url = NULL;
|
||||
char *url, *tmp;
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
|
||||
if (has_pack_index(sha1)) {
|
||||
ret = 0;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (http_is_verbose)
|
||||
fprintf(stderr, "Getting index for pack %s\n", hex);
|
||||
fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
|
||||
|
||||
end_url_with_slash(&buf, base_url);
|
||||
strbuf_addf(&buf, "objects/pack/pack-%s.idx", hex);
|
||||
strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
|
||||
url = strbuf_detach(&buf, NULL);
|
||||
|
||||
filename = sha1_pack_index_name(sha1);
|
||||
if (http_get_file(url, filename, 0) != HTTP_OK)
|
||||
ret = error("Unable to get pack index %s\n", url);
|
||||
strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
|
||||
tmp = strbuf_detach(&buf, NULL);
|
||||
|
||||
if (http_get_file(url, tmp, 0) != HTTP_OK) {
|
||||
error("Unable to get pack index %s\n", url);
|
||||
free(tmp);
|
||||
tmp = NULL;
|
||||
}
|
||||
|
||||
cleanup:
|
||||
free(hex);
|
||||
free(url);
|
||||
return ret;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
static int fetch_and_setup_pack_index(struct packed_git **packs_head,
|
||||
unsigned char *sha1, const char *base_url)
|
||||
{
|
||||
struct packed_git *new_pack;
|
||||
char *tmp_idx = NULL;
|
||||
int ret;
|
||||
|
||||
if (fetch_pack_index(sha1, base_url))
|
||||
if (has_pack_index(sha1)) {
|
||||
new_pack = parse_pack_index(sha1, NULL);
|
||||
if (!new_pack)
|
||||
return -1; /* parse_pack_index() already issued error message */
|
||||
goto add_pack;
|
||||
}
|
||||
|
||||
tmp_idx = fetch_pack_index(sha1, base_url);
|
||||
if (!tmp_idx)
|
||||
return -1;
|
||||
|
||||
new_pack = parse_pack_index(sha1);
|
||||
if (!new_pack)
|
||||
new_pack = parse_pack_index(sha1, tmp_idx);
|
||||
if (!new_pack) {
|
||||
unlink(tmp_idx);
|
||||
free(tmp_idx);
|
||||
|
||||
return -1; /* parse_pack_index() already issued error message */
|
||||
}
|
||||
|
||||
ret = verify_pack_index(new_pack);
|
||||
if (!ret) {
|
||||
close_pack_index(new_pack);
|
||||
ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
|
||||
}
|
||||
free(tmp_idx);
|
||||
if (ret)
|
||||
return -1;
|
||||
|
||||
add_pack:
|
||||
new_pack->next = *packs_head;
|
||||
*packs_head = new_pack;
|
||||
return 0;
|
||||
@ -1000,37 +1021,62 @@ void release_http_pack_request(struct http_pack_request *preq)
|
||||
|
||||
int finish_http_pack_request(struct http_pack_request *preq)
|
||||
{
|
||||
int ret;
|
||||
struct packed_git **lst;
|
||||
struct packed_git *p = preq->target;
|
||||
char *tmp_idx;
|
||||
struct child_process ip;
|
||||
const char *ip_argv[8];
|
||||
|
||||
preq->target->pack_size = ftell(preq->packfile);
|
||||
close_pack_index(p);
|
||||
|
||||
if (preq->packfile != NULL) {
|
||||
fclose(preq->packfile);
|
||||
preq->packfile = NULL;
|
||||
preq->slot->local = NULL;
|
||||
}
|
||||
|
||||
ret = move_temp_to_file(preq->tmpfile, preq->filename);
|
||||
if (ret)
|
||||
return ret;
|
||||
fclose(preq->packfile);
|
||||
preq->packfile = NULL;
|
||||
preq->slot->local = NULL;
|
||||
|
||||
lst = preq->lst;
|
||||
while (*lst != preq->target)
|
||||
while (*lst != p)
|
||||
lst = &((*lst)->next);
|
||||
*lst = (*lst)->next;
|
||||
|
||||
if (verify_pack(preq->target))
|
||||
return -1;
|
||||
install_packed_git(preq->target);
|
||||
tmp_idx = xstrdup(preq->tmpfile);
|
||||
strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
|
||||
".idx.temp");
|
||||
|
||||
ip_argv[0] = "index-pack";
|
||||
ip_argv[1] = "-o";
|
||||
ip_argv[2] = tmp_idx;
|
||||
ip_argv[3] = preq->tmpfile;
|
||||
ip_argv[4] = NULL;
|
||||
|
||||
memset(&ip, 0, sizeof(ip));
|
||||
ip.argv = ip_argv;
|
||||
ip.git_cmd = 1;
|
||||
ip.no_stdin = 1;
|
||||
ip.no_stdout = 1;
|
||||
|
||||
if (run_command(&ip)) {
|
||||
unlink(preq->tmpfile);
|
||||
unlink(tmp_idx);
|
||||
free(tmp_idx);
|
||||
return -1;
|
||||
}
|
||||
|
||||
unlink(sha1_pack_index_name(p->sha1));
|
||||
|
||||
if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
|
||||
|| move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
|
||||
free(tmp_idx);
|
||||
return -1;
|
||||
}
|
||||
|
||||
install_packed_git(p);
|
||||
free(tmp_idx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct http_pack_request *new_http_pack_request(
|
||||
struct packed_git *target, const char *base_url)
|
||||
{
|
||||
char *filename;
|
||||
long prev_posn = 0;
|
||||
char range[RANGE_HEADER_SIZE];
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
@ -1045,9 +1091,8 @@ struct http_pack_request *new_http_pack_request(
|
||||
sha1_to_hex(target->sha1));
|
||||
preq->url = strbuf_detach(&buf, NULL);
|
||||
|
||||
filename = sha1_pack_name(target->sha1);
|
||||
snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
|
||||
snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
|
||||
snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
|
||||
sha1_pack_name(target->sha1));
|
||||
preq->packfile = fopen(preq->tmpfile, "a");
|
||||
if (!preq->packfile) {
|
||||
error("Unable to open local file %s for pack",
|
||||
@ -1082,7 +1127,6 @@ struct http_pack_request *new_http_pack_request(
|
||||
return preq;
|
||||
|
||||
abort:
|
||||
free(filename);
|
||||
free(preq->url);
|
||||
free(preq);
|
||||
return NULL;
|
||||
@ -1137,7 +1181,6 @@ struct http_object_request *new_http_object_request(const char *base_url,
|
||||
freq->localfile = -1;
|
||||
|
||||
filename = sha1_file_name(sha1);
|
||||
snprintf(freq->filename, sizeof(freq->filename), "%s", filename);
|
||||
snprintf(freq->tmpfile, sizeof(freq->tmpfile),
|
||||
"%s.temp", filename);
|
||||
|
||||
@ -1166,8 +1209,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
|
||||
}
|
||||
|
||||
if (freq->localfile < 0) {
|
||||
error("Couldn't create temporary file %s for %s: %s",
|
||||
freq->tmpfile, freq->filename, strerror(errno));
|
||||
error("Couldn't create temporary file %s: %s",
|
||||
freq->tmpfile, strerror(errno));
|
||||
goto abort;
|
||||
}
|
||||
|
||||
@ -1214,8 +1257,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
|
||||
prev_posn = 0;
|
||||
lseek(freq->localfile, 0, SEEK_SET);
|
||||
if (ftruncate(freq->localfile, 0) < 0) {
|
||||
error("Couldn't truncate temporary file %s for %s: %s",
|
||||
freq->tmpfile, freq->filename, strerror(errno));
|
||||
error("Couldn't truncate temporary file %s: %s",
|
||||
freq->tmpfile, strerror(errno));
|
||||
goto abort;
|
||||
}
|
||||
}
|
||||
@ -1291,7 +1334,7 @@ int finish_http_object_request(struct http_object_request *freq)
|
||||
return -1;
|
||||
}
|
||||
freq->rename =
|
||||
move_temp_to_file(freq->tmpfile, freq->filename);
|
||||
move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
|
||||
|
||||
return freq->rename;
|
||||
}
|
||||
|
||||
2
http.h
2
http.h
@ -153,7 +153,6 @@ struct http_pack_request
|
||||
struct packed_git *target;
|
||||
struct packed_git **lst;
|
||||
FILE *packfile;
|
||||
char filename[PATH_MAX];
|
||||
char tmpfile[PATH_MAX];
|
||||
struct curl_slist *range_header;
|
||||
struct active_request_slot *slot;
|
||||
@ -168,7 +167,6 @@ extern void release_http_pack_request(struct http_pack_request *preq);
|
||||
struct http_object_request
|
||||
{
|
||||
char *url;
|
||||
char filename[PATH_MAX];
|
||||
char tmpfile[PATH_MAX];
|
||||
int localfile;
|
||||
CURLcode curl_result;
|
||||
|
||||
15
pack-check.c
15
pack-check.c
@ -133,14 +133,13 @@ static int verify_packfile(struct packed_git *p,
|
||||
return err;
|
||||
}
|
||||
|
||||
int verify_pack(struct packed_git *p)
|
||||
int verify_pack_index(struct packed_git *p)
|
||||
{
|
||||
off_t index_size;
|
||||
const unsigned char *index_base;
|
||||
git_SHA_CTX ctx;
|
||||
unsigned char sha1[20];
|
||||
int err = 0;
|
||||
struct pack_window *w_curs = NULL;
|
||||
|
||||
if (open_pack_index(p))
|
||||
return error("packfile %s index not opened", p->pack_name);
|
||||
@ -154,8 +153,18 @@ int verify_pack(struct packed_git *p)
|
||||
if (hashcmp(sha1, index_base + index_size - 20))
|
||||
err = error("Packfile index for %s SHA1 mismatch",
|
||||
p->pack_name);
|
||||
return err;
|
||||
}
|
||||
|
||||
int verify_pack(struct packed_git *p)
|
||||
{
|
||||
int err = 0;
|
||||
struct pack_window *w_curs = NULL;
|
||||
|
||||
err |= verify_pack_index(p);
|
||||
if (!p->index_data)
|
||||
return -1;
|
||||
|
||||
/* Verify pack file */
|
||||
err |= verify_packfile(p, &w_curs);
|
||||
unuse_pack(&w_curs);
|
||||
|
||||
|
||||
1
pack.h
1
pack.h
@ -57,6 +57,7 @@ struct pack_idx_entry {
|
||||
|
||||
extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
|
||||
extern int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr);
|
||||
extern int verify_pack_index(struct packed_git *);
|
||||
extern int verify_pack(struct packed_git *);
|
||||
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t, unsigned char *, off_t);
|
||||
extern char *index_pack_lockfile(int fd);
|
||||
|
||||
14
sha1_file.c
14
sha1_file.c
@ -599,6 +599,14 @@ void unuse_pack(struct pack_window **w_cursor)
|
||||
}
|
||||
}
|
||||
|
||||
void close_pack_index(struct packed_git *p)
|
||||
{
|
||||
if (p->index_data) {
|
||||
munmap((void *)p->index_data, p->index_size);
|
||||
p->index_data = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This is used by git-repack in case a newly created pack happens to
|
||||
* contain the same set of objects as an existing one. In that case
|
||||
@ -620,8 +628,7 @@ void free_pack_by_name(const char *pack_name)
|
||||
close_pack_windows(p);
|
||||
if (p->pack_fd != -1)
|
||||
close(p->pack_fd);
|
||||
if (p->index_data)
|
||||
munmap((void *)p->index_data, p->index_size);
|
||||
close_pack_index(p);
|
||||
free(p->bad_object_sha1);
|
||||
*pp = p->next;
|
||||
free(p);
|
||||
@ -831,9 +838,8 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
|
||||
return p;
|
||||
}
|
||||
|
||||
struct packed_git *parse_pack_index(unsigned char *sha1)
|
||||
struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
|
||||
{
|
||||
const char *idx_path = sha1_pack_index_name(sha1);
|
||||
const char *path = sha1_pack_name(sha1);
|
||||
struct packed_git *p = alloc_packed_git(strlen(path) + 1);
|
||||
|
||||
|
||||
@ -55,12 +55,43 @@ test_expect_success 'http remote detects correct HEAD' '
|
||||
|
||||
test_expect_success 'fetch packed objects' '
|
||||
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
|
||||
cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
|
||||
git --bare repack &&
|
||||
git --bare prune-packed &&
|
||||
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
|
||||
git --bare repack &&
|
||||
git --bare prune-packed
|
||||
) &&
|
||||
git clone $HTTPD_URL/dumb/repo_pack.git
|
||||
'
|
||||
|
||||
test_expect_success 'fetch notices corrupt pack' '
|
||||
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
|
||||
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
|
||||
p=`ls objects/pack/pack-*.pack` &&
|
||||
chmod u+w $p &&
|
||||
printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc
|
||||
) &&
|
||||
mkdir repo_bad1.git &&
|
||||
(cd repo_bad1.git &&
|
||||
git --bare init &&
|
||||
test_must_fail git --bare fetch $HTTPD_URL/dumb/repo_bad1.git &&
|
||||
test 0 = `ls objects/pack/pack-*.pack | wc -l`
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'fetch notices corrupt idx' '
|
||||
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
|
||||
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
|
||||
p=`ls objects/pack/pack-*.idx` &&
|
||||
chmod u+w $p &&
|
||||
printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc
|
||||
) &&
|
||||
mkdir repo_bad2.git &&
|
||||
(cd repo_bad2.git &&
|
||||
git --bare init &&
|
||||
test_must_fail git --bare fetch $HTTPD_URL/dumb/repo_bad2.git &&
|
||||
test 0 = `ls objects/pack | wc -l`
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'did not use upload-pack service' '
|
||||
grep '/git-upload-pack' <"$HTTPD_ROOT_PATH"/access.log >act
|
||||
: >exp
|
||||
|
||||
Loading…
Reference in New Issue
Block a user