[PATCH] Teach read_sha1_file() and friends about packed git object store.
GIT_OBJECT_DIRECTORY and GIT_ALTERNATE_OBJECT_DIRECTORIES can have the "pack" subdirectory that houses "packed GIT" files produced by git-pack-objects (e.g. .git/objects/pack/foo.pack and .git/objects/pack/foo.idx; always store them as pairs). The following functions in sha1_file.c can then read object contents from such packed file: - sha1_object_info() - has_sha1_file() - read_sha1_file() Signed-off-by: Junio C Hamano <junkio@cox.net> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
Junio C Hamano
Linus Torvalds
parent
36e4d74a21
commit
1f688557c0
422
sha1_file.c
422
sha1_file.c
@ -6,7 +6,10 @@
|
||||
* This handles basic git sha1 object files - packing, unpacking,
|
||||
* creation etc.
|
||||
*/
|
||||
#include <sys/types.h>
|
||||
#include <dirent.h>
|
||||
#include "cache.h"
|
||||
#include "delta.h"
|
||||
|
||||
#ifndef O_NOATIME
|
||||
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
|
||||
@ -265,6 +268,184 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#define PACK_MAX_SZ (1<<26)
|
||||
static int pack_used_ctr;
|
||||
static unsigned long pack_mapped;
|
||||
static struct packed_git {
|
||||
struct packed_git *next;
|
||||
unsigned long index_size;
|
||||
unsigned long pack_size;
|
||||
unsigned int *index_base;
|
||||
void *pack_base;
|
||||
unsigned int pack_last_used;
|
||||
char pack_name[0]; /* something like ".git/objects/pack/xxxxx.pack" */
|
||||
} *packed_git;
|
||||
|
||||
struct pack_entry {
|
||||
unsigned int offset;
|
||||
unsigned char sha1[20];
|
||||
struct packed_git *p;
|
||||
};
|
||||
|
||||
static int check_packed_git_idx(const char *path, unsigned long *idx_size_,
|
||||
void **idx_map_)
|
||||
{
|
||||
void *idx_map;
|
||||
unsigned int *index;
|
||||
unsigned long idx_size;
|
||||
int nr, i;
|
||||
int fd = open(path, O_RDONLY);
|
||||
struct stat st;
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
if (fstat(fd, &st)) {
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
idx_size = st.st_size;
|
||||
idx_map = mmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||
close(fd);
|
||||
if (idx_map == MAP_FAILED)
|
||||
return -1;
|
||||
|
||||
index = idx_map;
|
||||
|
||||
/* check index map */
|
||||
if (idx_size < 4*256 + 20)
|
||||
return error("index file too small");
|
||||
nr = 0;
|
||||
for (i = 0; i < 256; i++) {
|
||||
unsigned int n = ntohl(index[i]);
|
||||
if (n < nr)
|
||||
return error("non-monotonic index");
|
||||
nr = n;
|
||||
}
|
||||
|
||||
/*
|
||||
* Total size:
|
||||
* - 256 index entries 4 bytes each
|
||||
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
|
||||
* - 20-byte SHA1 of the packfile
|
||||
* - 20-byte SHA1 file checksum
|
||||
*/
|
||||
if (idx_size != 4*256 + nr * 24 + 20 + 20)
|
||||
return error("wrong index file size");
|
||||
|
||||
*idx_map_ = idx_map;
|
||||
*idx_size_ = idx_size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void unuse_one_packed_git(void)
|
||||
{
|
||||
/* NOTYET */
|
||||
}
|
||||
|
||||
static int use_packed_git(struct packed_git *p)
|
||||
{
|
||||
if (!p->pack_base) {
|
||||
int fd;
|
||||
struct stat st;
|
||||
void *map;
|
||||
|
||||
pack_mapped += p->pack_size;
|
||||
while (PACK_MAX_SZ < pack_mapped)
|
||||
unuse_one_packed_git();
|
||||
fd = open(p->pack_name, O_RDONLY);
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
if (fstat(fd, &st)) {
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
if (st.st_size != p->pack_size)
|
||||
return -1;
|
||||
map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||
close(fd);
|
||||
if (map == MAP_FAILED)
|
||||
return -1;
|
||||
p->pack_base = map;
|
||||
}
|
||||
p->pack_last_used = pack_used_ctr++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct packed_git *add_packed_git(char *path, int path_len)
|
||||
{
|
||||
struct stat st;
|
||||
struct packed_git *p;
|
||||
unsigned long idx_size;
|
||||
void *idx_map;
|
||||
|
||||
if (check_packed_git_idx(path, &idx_size, &idx_map))
|
||||
return NULL;
|
||||
|
||||
/* do we have a corresponding .pack file? */
|
||||
strcpy(path + path_len - 4, ".pack");
|
||||
if (stat(path, &st) || !S_ISREG(st.st_mode)) {
|
||||
munmap(idx_map, idx_size);
|
||||
return NULL;
|
||||
}
|
||||
/* ok, it looks sane as far as we can check without
|
||||
* actually mapping the pack file.
|
||||
*/
|
||||
p = xmalloc(sizeof(*p) + path_len + 2);
|
||||
strcpy(p->pack_name, path);
|
||||
p->index_size = idx_size;
|
||||
p->pack_size = st.st_size;
|
||||
p->index_base = idx_map;
|
||||
p->next = NULL;
|
||||
p->pack_last_used = 0;
|
||||
return p;
|
||||
}
|
||||
|
||||
static void prepare_packed_git_one(char *objdir)
|
||||
{
|
||||
char path[PATH_MAX];
|
||||
int len;
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
|
||||
sprintf(path, "%s/pack", objdir);
|
||||
len = strlen(path);
|
||||
dir = opendir(path);
|
||||
if (!dir)
|
||||
return;
|
||||
path[len++] = '/';
|
||||
while ((de = readdir(dir)) != NULL) {
|
||||
int namelen = strlen(de->d_name);
|
||||
struct packed_git *p;
|
||||
|
||||
if (strcmp(de->d_name + namelen - 4, ".idx"))
|
||||
continue;
|
||||
|
||||
/* we have .idx. Is it a file we can map? */
|
||||
strcpy(path + len, de->d_name);
|
||||
p = add_packed_git(path, len + namelen);
|
||||
if (!p)
|
||||
continue;
|
||||
p->next = packed_git;
|
||||
packed_git = p;
|
||||
}
|
||||
}
|
||||
|
||||
static void prepare_packed_git(void)
|
||||
{
|
||||
int i;
|
||||
static int run_once = 0;
|
||||
|
||||
if (run_once++)
|
||||
return;
|
||||
|
||||
prepare_packed_git_one(get_object_directory());
|
||||
if (!alt_odb)
|
||||
prepare_alt_odb();
|
||||
for (i = 0; alt_odb[i].base != NULL; i++) {
|
||||
alt_odb[i].name[0] = 0;
|
||||
prepare_packed_git_one(alt_odb[i].base);
|
||||
}
|
||||
}
|
||||
|
||||
int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
|
||||
{
|
||||
char header[100];
|
||||
@ -278,7 +459,9 @@ int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long siz
|
||||
return memcmp(sha1, real_sha1, 20) ? -1 : 0;
|
||||
}
|
||||
|
||||
void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
||||
static void *map_sha1_file_internal(const unsigned char *sha1,
|
||||
unsigned long *size,
|
||||
int say_error)
|
||||
{
|
||||
struct stat st;
|
||||
void *map;
|
||||
@ -286,7 +469,8 @@ void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
||||
char *filename = find_sha1_file(sha1, &st);
|
||||
|
||||
if (!filename) {
|
||||
error("cannot map sha1 file %s", sha1_to_hex(sha1));
|
||||
if (say_error)
|
||||
error("cannot map sha1 file %s", sha1_to_hex(sha1));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -300,11 +484,14 @@ void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
||||
break;
|
||||
/* Fallthrough */
|
||||
case 0:
|
||||
perror(filename);
|
||||
if (say_error)
|
||||
perror(filename);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* If it failed once, it will probably fail again. Stop using O_NOATIME */
|
||||
/* If it failed once, it will probably fail again.
|
||||
* Stop using O_NOATIME
|
||||
*/
|
||||
sha1_file_open_flag = 0;
|
||||
}
|
||||
map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||
@ -315,6 +502,11 @@ void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
||||
return map;
|
||||
}
|
||||
|
||||
void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
|
||||
{
|
||||
return map_sha1_file_internal(sha1, size, 1);
|
||||
}
|
||||
|
||||
int unpack_sha1_header(z_stream *stream, void *map, unsigned long mapsize, void *buffer, unsigned long size)
|
||||
{
|
||||
/* Get the data stream */
|
||||
@ -409,6 +601,191 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l
|
||||
return unpack_sha1_rest(&stream, hdr, *size);
|
||||
}
|
||||
|
||||
/* Returns 0 on fast-path success, returns 1 on deltified
|
||||
* and need to unpack to see info.
|
||||
*/
|
||||
static int packed_object_info(struct pack_entry *entry,
|
||||
char *type, unsigned long *sizep)
|
||||
{
|
||||
struct packed_git *p = entry->p;
|
||||
unsigned long offset, size, left;
|
||||
unsigned char *pack;
|
||||
|
||||
offset = entry->offset;
|
||||
if (p->pack_size - 5 < offset)
|
||||
die("object offset outside of pack file");
|
||||
pack = p->pack_base + offset;
|
||||
size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
|
||||
left = p->pack_size - offset - 5;
|
||||
switch (*pack) {
|
||||
case 'D':
|
||||
return 1;
|
||||
break;
|
||||
case 'C':
|
||||
strcpy(type, "commit");
|
||||
break;
|
||||
case 'T':
|
||||
strcpy(type, "tree");
|
||||
break;
|
||||
case 'B':
|
||||
strcpy(type, "blob");
|
||||
break;
|
||||
default:
|
||||
die("corrupted pack file");
|
||||
}
|
||||
*sizep = size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* forward declaration for a mutually recursive function */
|
||||
static void *unpack_entry(struct pack_entry *, char *, unsigned long *);
|
||||
|
||||
static void *unpack_delta_entry(unsigned char *base_sha1,
|
||||
unsigned long delta_size,
|
||||
unsigned long left,
|
||||
char *type,
|
||||
unsigned long *sizep)
|
||||
{
|
||||
void *data, *delta_data, *result, *base;
|
||||
unsigned long data_size, result_size, base_size;
|
||||
z_stream stream;
|
||||
int st;
|
||||
|
||||
if (left < 20)
|
||||
die("truncated pack file");
|
||||
data = base_sha1 + 20;
|
||||
data_size = left - 20;
|
||||
delta_data = xmalloc(delta_size);
|
||||
|
||||
memset(&stream, 0, sizeof(stream));
|
||||
|
||||
stream.next_in = data;
|
||||
stream.avail_in = data_size;
|
||||
stream.next_out = delta_data;
|
||||
stream.avail_out = delta_size;
|
||||
|
||||
inflateInit(&stream);
|
||||
st = inflate(&stream, Z_FINISH);
|
||||
inflateEnd(&stream);
|
||||
if ((st != Z_STREAM_END) || stream.total_out != delta_size)
|
||||
die("delta data unpack failed");
|
||||
|
||||
/* This may recursively unpack the base, which is what we want */
|
||||
base = read_sha1_file(base_sha1, type, &base_size);
|
||||
if (!base)
|
||||
die("failed to read delta-pack base object %s",
|
||||
sha1_to_hex(base_sha1));
|
||||
result = patch_delta(base, base_size,
|
||||
delta_data, delta_size,
|
||||
&result_size);
|
||||
if (!result)
|
||||
die("failed to apply delta");
|
||||
free(delta_data);
|
||||
free(base);
|
||||
*sizep = result_size;
|
||||
return result;
|
||||
}
|
||||
|
||||
static void *unpack_non_delta_entry(unsigned char *data,
|
||||
unsigned long size,
|
||||
unsigned long left)
|
||||
{
|
||||
int st;
|
||||
z_stream stream;
|
||||
char *buffer;
|
||||
|
||||
buffer = xmalloc(size + 1);
|
||||
buffer[size] = 0;
|
||||
memset(&stream, 0, sizeof(stream));
|
||||
stream.next_in = data;
|
||||
stream.avail_in = left;
|
||||
stream.next_out = buffer;
|
||||
stream.avail_out = size;
|
||||
|
||||
inflateInit(&stream);
|
||||
st = inflate(&stream, Z_FINISH);
|
||||
inflateEnd(&stream);
|
||||
if ((st != Z_STREAM_END) || stream.total_out != size) {
|
||||
free(buffer);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static void *unpack_entry(struct pack_entry *entry,
|
||||
char *type, unsigned long *sizep)
|
||||
{
|
||||
struct packed_git *p = entry->p;
|
||||
unsigned long offset, size, left;
|
||||
unsigned char *pack;
|
||||
|
||||
offset = entry->offset;
|
||||
if (p->pack_size - 5 < offset)
|
||||
die("object offset outside of pack file");
|
||||
|
||||
if (use_packed_git(p))
|
||||
die("cannot map packed file");
|
||||
|
||||
pack = p->pack_base + offset;
|
||||
size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
|
||||
left = p->pack_size - offset - 5;
|
||||
switch (*pack) {
|
||||
case 'D':
|
||||
return unpack_delta_entry(pack+5, size, left, type, sizep);
|
||||
case 'C':
|
||||
strcpy(type, "commit");
|
||||
break;
|
||||
case 'T':
|
||||
strcpy(type, "tree");
|
||||
break;
|
||||
case 'B':
|
||||
strcpy(type, "blob");
|
||||
break;
|
||||
default:
|
||||
die("corrupted pack file");
|
||||
}
|
||||
*sizep = size;
|
||||
return unpack_non_delta_entry(pack+5, size, left);
|
||||
}
|
||||
|
||||
static int find_pack_entry_1(const unsigned char *sha1,
|
||||
struct pack_entry *e, struct packed_git *p)
|
||||
{
|
||||
int *level1_ofs = p->index_base;
|
||||
int hi = ntohl(level1_ofs[*sha1]);
|
||||
int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
|
||||
void *index = p->index_base + 256;
|
||||
|
||||
do {
|
||||
int mi = (lo + hi) / 2;
|
||||
int cmp = memcmp(index + 24 * mi + 4, sha1, 20);
|
||||
if (!cmp) {
|
||||
e->offset = ntohl(*((int*)(index + 24 * mi)));
|
||||
memcpy(e->sha1, sha1, 20);
|
||||
e->p = p;
|
||||
return 1;
|
||||
}
|
||||
if (cmp > 0)
|
||||
hi = mi;
|
||||
else
|
||||
lo = mi+1;
|
||||
} while (lo < hi);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
|
||||
{
|
||||
struct packed_git *p;
|
||||
prepare_packed_git();
|
||||
|
||||
for (p = packed_git; p; p = p->next) {
|
||||
if (find_pack_entry_1(sha1, e, p))
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep)
|
||||
{
|
||||
int status;
|
||||
@ -417,9 +794,19 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep
|
||||
z_stream stream;
|
||||
char hdr[128];
|
||||
|
||||
map = map_sha1_file(sha1, &mapsize);
|
||||
if (!map)
|
||||
return error("unable to map %s", sha1_to_hex(sha1));
|
||||
map = map_sha1_file_internal(sha1, &mapsize, 0);
|
||||
if (!map) {
|
||||
struct pack_entry e;
|
||||
|
||||
if (!find_pack_entry(sha1, &e))
|
||||
return error("unable to find %s", sha1_to_hex(sha1));
|
||||
if (!packed_object_info(&e, type, sizep))
|
||||
return 0;
|
||||
/* sheesh */
|
||||
map = unpack_entry(&e, type, sizep);
|
||||
free(map);
|
||||
return (map == NULL) ? 0 : -1;
|
||||
}
|
||||
if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
|
||||
status = error("unable to unpack %s header",
|
||||
sha1_to_hex(sha1));
|
||||
@ -434,18 +821,29 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep
|
||||
return status;
|
||||
}
|
||||
|
||||
static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned long *size)
|
||||
{
|
||||
struct pack_entry e;
|
||||
|
||||
if (!find_pack_entry(sha1, &e)) {
|
||||
error("cannot read sha1_file for %s", sha1_to_hex(sha1));
|
||||
return NULL;
|
||||
}
|
||||
return unpack_entry(&e, type, size);
|
||||
}
|
||||
|
||||
void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size)
|
||||
{
|
||||
unsigned long mapsize;
|
||||
void *map, *buf;
|
||||
|
||||
map = map_sha1_file(sha1, &mapsize);
|
||||
map = map_sha1_file_internal(sha1, &mapsize, 0);
|
||||
if (map) {
|
||||
buf = unpack_sha1_file(map, mapsize, type, size);
|
||||
munmap(map, mapsize);
|
||||
return buf;
|
||||
}
|
||||
return NULL;
|
||||
return read_packed_sha1(sha1, type, size);
|
||||
}
|
||||
|
||||
void *read_object_with_reference(const unsigned char *sha1,
|
||||
@ -664,7 +1062,11 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd)
|
||||
int has_sha1_file(const unsigned char *sha1)
|
||||
{
|
||||
struct stat st;
|
||||
return !!find_sha1_file(sha1, &st);
|
||||
struct pack_entry e;
|
||||
|
||||
if (find_sha1_file(sha1, &st))
|
||||
return 1;
|
||||
return find_pack_entry(sha1, &e);
|
||||
}
|
||||
|
||||
int index_fd(unsigned char *sha1, int fd, struct stat *st)
|
||||
|
||||
@ -19,10 +19,19 @@ test_expect_success \
|
||||
git-update-cache --add $i || exit
|
||||
done &&
|
||||
cat c >d && echo foo >>d && git-update-cache --add d &&
|
||||
tree=`git-write-tree` && {
|
||||
tree=`git-write-tree` &&
|
||||
commit=`git-commit-tree $tree </dev/null` && {
|
||||
echo $tree &&
|
||||
echo $commit &&
|
||||
git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/"
|
||||
} >obj-list'
|
||||
} >obj-list && {
|
||||
git-diff-tree --root -p $commit &&
|
||||
while read object
|
||||
do
|
||||
t=`git-cat-file -t $object` &&
|
||||
git-cat-file $t $object || exit 1
|
||||
done <obj-list
|
||||
} >expect'
|
||||
|
||||
test_expect_success \
|
||||
'pack without delta' \
|
||||
@ -82,4 +91,39 @@ test_expect_success \
|
||||
done'
|
||||
cd $TRASH
|
||||
|
||||
rm -fr .git2
|
||||
mkdir .git2
|
||||
|
||||
test_expect_success \
|
||||
'use packed objects' \
|
||||
'GIT_OBJECT_DIRECTORY=.git2/objects &&
|
||||
export GIT_OBJECT_DIRECTORY &&
|
||||
git-init-db &&
|
||||
mkdir .git2/objects/pack &&
|
||||
cp test-1.pack test-1.idx .git2/objects/pack && {
|
||||
git-diff-tree --root -p $commit &&
|
||||
while read object
|
||||
do
|
||||
t=`git-cat-file -t $object` &&
|
||||
git-cat-file $t $object || exit 1
|
||||
done <obj-list
|
||||
} >current &&
|
||||
diff expect current'
|
||||
|
||||
|
||||
test_expect_success \
|
||||
'use packed deltified objects' \
|
||||
'GIT_OBJECT_DIRECTORY=.git2/objects &&
|
||||
export GIT_OBJECT_DIRECTORY &&
|
||||
rm -f .git2/objects/pack/test-?.idx &&
|
||||
cp test-2.pack test-2.idx .git2/objects/pack && {
|
||||
git-diff-tree --root -p $commit &&
|
||||
while read object
|
||||
do
|
||||
t=`git-cat-file -t $object` &&
|
||||
git-cat-file $t $object || exit 1
|
||||
done <obj-list
|
||||
} >current &&
|
||||
diff expect current'
|
||||
|
||||
test_done
|
||||
|
||||
Loading…
Reference in New Issue
Block a user