1687150b5d
The hashfile API is useful for generating files that include a trailing
hash of the file's contents up to that point. Using such a hash is
helpful for verifying the file for corruption-at-rest, such as a faulty
drive causing flipped bits.
Git's index file includes this trailing hash, so it uses a 'struct
hashfile' to handle the I/O to the file. This was very convenient to
allow using the hashfile methods during these operations.
However, hashing the file contents during write comes at a performance
penalty. It's slower to hash the bytes on their way to the disk than
without that step. This problem is made worse by the replacement of
hardware-accelerated SHA1 computations with the software-based sha1dc
computation.
This write cost is significant, and the checksum capability is likely
not worth that cost for such a short-lived file. The index is rewritten
frequently and the only time the checksum is checked is during 'git
fsck'. Thus, it would be helpful to allow a user to opt-out of the hash
computation.
We first need to allow Git to opt-out of the hash computation in the
hashfile API. The buffered writes of the API are still helpful, so it
makes sense to make the change here.
Introduce a new 'skip_hash' option to 'struct hashfile'. When set, the
update_fn and final_fn members of the_hash_algo are skipped. When
finalizing the hashfile, the trailing hash is replaced with the null
hash.
This use of a trailing null hash would be desireable in either case,
since we do not want to special case a file format to have a different
length depending on whether it was hashed or not. When the final bytes
of a file are all zero, we can infer that it was written without
hashing, and thus that verification is not available as a check for file
consistency. This also means that we could easily toggle hashing for any
file format we desire.
A version of this patch has existed in the microsoft/git fork since
2017 [1] (the linked commit was rebased in 2018, but the original dates
back to January 2017). Here, the change to make the index use this fast
path is delayed until a later change.
[1] 21fed2d914
Co-authored-by: Kevin Willford <kewillf@microsoft.com>
Signed-off-by: Kevin Willford <kewillf@microsoft.com>
Signed-off-by: Derrick Stolee <derrickstolee@github.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
86 lines
2.1 KiB
C
86 lines
2.1 KiB
C
#ifndef CSUM_FILE_H
|
|
#define CSUM_FILE_H
|
|
|
|
#include "cache.h"
|
|
#include "hash.h"
|
|
|
|
struct progress;
|
|
|
|
/* A SHA1-protected file */
|
|
struct hashfile {
|
|
int fd;
|
|
int check_fd;
|
|
unsigned int offset;
|
|
git_hash_ctx ctx;
|
|
off_t total;
|
|
struct progress *tp;
|
|
const char *name;
|
|
int do_crc;
|
|
uint32_t crc32;
|
|
size_t buffer_len;
|
|
unsigned char *buffer;
|
|
unsigned char *check_buffer;
|
|
|
|
/**
|
|
* If non-zero, skip_hash indicates that we should
|
|
* not actually compute the hash for this hashfile and
|
|
* instead only use it as a buffered write.
|
|
*/
|
|
int skip_hash;
|
|
};
|
|
|
|
/* Checkpoint */
|
|
struct hashfile_checkpoint {
|
|
off_t offset;
|
|
git_hash_ctx ctx;
|
|
};
|
|
|
|
void hashfile_checkpoint(struct hashfile *, struct hashfile_checkpoint *);
|
|
int hashfile_truncate(struct hashfile *, struct hashfile_checkpoint *);
|
|
|
|
/* finalize_hashfile flags */
|
|
#define CSUM_CLOSE 1
|
|
#define CSUM_FSYNC 2
|
|
#define CSUM_HASH_IN_STREAM 4
|
|
|
|
struct hashfile *hashfd(int fd, const char *name);
|
|
struct hashfile *hashfd_check(const char *name);
|
|
struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp);
|
|
int finalize_hashfile(struct hashfile *, unsigned char *, enum fsync_component, unsigned int);
|
|
void hashwrite(struct hashfile *, const void *, unsigned int);
|
|
void hashflush(struct hashfile *f);
|
|
void crc32_begin(struct hashfile *);
|
|
uint32_t crc32_end(struct hashfile *);
|
|
|
|
/* Verify checksum validity while reading. Returns non-zero on success. */
|
|
int hashfile_checksum_valid(const unsigned char *data, size_t len);
|
|
|
|
/*
|
|
* Returns the total number of bytes fed to the hashfile so far (including ones
|
|
* that have not been written out to the descriptor yet).
|
|
*/
|
|
static inline off_t hashfile_total(struct hashfile *f)
|
|
{
|
|
return f->total + f->offset;
|
|
}
|
|
|
|
static inline void hashwrite_u8(struct hashfile *f, uint8_t data)
|
|
{
|
|
hashwrite(f, &data, sizeof(data));
|
|
}
|
|
|
|
static inline void hashwrite_be32(struct hashfile *f, uint32_t data)
|
|
{
|
|
data = htonl(data);
|
|
hashwrite(f, &data, sizeof(data));
|
|
}
|
|
|
|
static inline size_t hashwrite_be64(struct hashfile *f, uint64_t data)
|
|
{
|
|
data = htonll(data);
|
|
hashwrite(f, &data, sizeof(data));
|
|
return sizeof(data);
|
|
}
|
|
|
|
#endif
|