undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'ns/core-fsyncmethod'

Browse Source 
Replace core.fsyncObjectFiles with two new configuration variables,
core.fsync and core.fsyncMethod.

* ns/core-fsyncmethod:
  core.fsync: documentation and user-friendly aggregate options
  core.fsync: new option to harden the index
  core.fsync: add configuration parsing
  core.fsync: introduce granular fsync control infrastructure
  core.fsyncmethod: add writeout-only mode
  wrapper: make inclusion of Windows csprng header tightly scoped
This commit is contained in:
Junio C Hamano 2022-03-25 16:38:24 -07:00
commit eb804cd405
26 changed files with 444 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
Documentation/config/core.txt
View File

@ -547,13 +547,63 @@ core.whitespace::
is relevant for `indent-with-non-tab` and when Git fixes `tab-in-indent`
errors. The default tab width is 8. Allowed values are 1 to 63.
core.fsync::
A comma-separated list of components of the repository that
should be hardened via the core.fsyncMethod when created or
modified. You can disable hardening of any component by
prefixing it with a '-'. Items that are not hardened may be
lost in the event of an unclean system shutdown. Unless you
have special requirements, it is recommended that you leave
this option empty or pick one of `committed`, `added`,
or `all`.
+
When this configuration is encountered, the set of components starts with
the platform default value, disabled components are removed, and additional
components are added. `none` resets the state so that the platform default
is ignored.
+
The empty string resets the fsync configuration to the platform
default. The default on most platforms is equivalent to
`core.fsync=committed,-loose-object`, which has good performance,
but risks losing recent work in the event of an unclean system shutdown.
+
* `none` clears the set of fsynced components.
* `loose-object` hardens objects added to the repo in loose-object form.
* `pack` hardens objects added to the repo in packfile form.
* `pack-metadata` hardens packfile bitmaps and indexes.
* `commit-graph` hardens the commit graph file.
* `index` hardens the index when it is modified.
* `objects` is an aggregate option that is equivalent to
`loose-object,pack`.
* `derived-metadata` is an aggregate option that is equivalent to
`pack-metadata,commit-graph`.
* `committed` is an aggregate option that is currently equivalent to
`objects`. This mode sacrifices some performance to ensure that work
that is committed to the repository with `git commit` or similar commands
is hardened.
* `added` is an aggregate option that is currently equivalent to
`committed,index`. This mode sacrifices additional performance to
ensure that the results of commands like `git add` and similar operations
are hardened.
* `all` is an aggregate option that syncs all individual components above.
core.fsyncMethod::
A value indicating the strategy Git will use to harden repository data
using fsync and related primitives.
+
* `fsync` uses the fsync() system call or platform equivalents.
* `writeout-only` issues pagecache writeback requests, but depending on the
filesystem and storage hardware, data added to the repository may not be
durable in the event of a system crash. This is the default mode on macOS.
core.fsyncObjectFiles::
This boolean will enable 'fsync()' when writing object files.
This setting is deprecated. Use core.fsync instead.
+
This is a total waste of time and effort on a filesystem that orders
data writes properly, but can be useful for filesystems that do not use
journalling (traditional UNIX filesystems) or that only journal metadata
and not file contents (OS X's HFS+, or Linux ext3 with "data=writeback").
This setting affects data added to the Git repository in loose-object
form. When set to true, Git will issue an fsync or similar system call
to flush caches so that loose-objects remain consistent in the face
of a unclean system shutdown.
core.preloadIndex::
Enable parallel index preload for operations like 'git diff'

6
Makefile
View File

@ -414,6 +414,8 @@ include shared.mak
#
# Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC.
#
# Define HAVE_SYNC_FILE_RANGE if your platform has sync_file_range.
#
# Define NEEDS_LIBRT if your platform requires linking with librt (glibc version
# before 2.17) for clock_gettime and CLOCK_MONOTONIC.
#
@ -1918,6 +1920,10 @@ ifdef HAVE_CLOCK_MONOTONIC
BASIC_CFLAGS += -DHAVE_CLOCK_MONOTONIC
endif
ifdef HAVE_SYNC_FILE_RANGE
BASIC_CFLAGS += -DHAVE_SYNC_FILE_RANGE
endif
ifdef NEEDS_LIBRT
EXTLIBS += -lrt
endif

2
builtin/fast-import.c
View File

@ -865,7 +865,7 @@ static void end_packfile(void)
struct tag *t;
close_pack_windows(pack_data);
finalize_hashfile(pack_file, cur_pack_oid.hash, 0);
finalize_hashfile(pack_file, cur_pack_oid.hash, FSYNC_COMPONENT_PACK, 0);
fixup_pack_header_footer(pack_data->pack_fd, pack_data->hash,
pack_data->pack_name, object_count,
cur_pack_oid.hash, pack_size);

4
builtin/index-pack.c
View File

@ -1291,7 +1291,7 @@ static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned cha
nr_objects - nr_objects_initial);
stop_progress_msg(&progress, msg.buf);
strbuf_release(&msg);
finalize_hashfile(f, tail_hash, 0);
finalize_hashfile(f, tail_hash, FSYNC_COMPONENT_PACK, 0);
hashcpy(read_hash, pack_hash);
fixup_pack_header_footer(output_fd, pack_hash,
curr_pack, nr_objects,
@ -1513,7 +1513,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
if (!from_stdin) {
close(input_fd);
} else {
fsync_or_die(output_fd, curr_pack_name);
fsync_component_or_die(FSYNC_COMPONENT_PACK, output_fd, curr_pack_name);
err = close(output_fd);
if (err)
die_errno(_("error while closing pack file"));

24
builtin/pack-objects.c
View File

@ -1199,16 +1199,26 @@ static void write_pack_file(void)
display_progress(progress_state, written);
}
/*
* Did we write the wrong # entries in the header?
* If so, rewrite it like in fast-import
*/
if (pack_to_stdout) {
finalize_hashfile(f, hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE);
/*
* We never fsync when writing to stdout since we may
* not be writing to an actual pack file. For instance,
* the upload-pack code passes a pipe here. Calling
* fsync on a pipe results in unnecessary
* synchronization with the reader on some platforms.
*/
finalize_hashfile(f, hash, FSYNC_COMPONENT_NONE,
CSUM_HASH_IN_STREAM | CSUM_CLOSE);
} else if (nr_written == nr_remaining) {
finalize_hashfile(f, hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK,
CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
} else {
int fd = finalize_hashfile(f, hash, 0);
/*
* If we wrote the wrong number of entries in the
* header, rewrite it like in fast-import.
*/
int fd = finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK, 0);
fixup_pack_header_footer(fd, hash, pack_tmp_name,
nr_written, hash, offset);
close(fd);

5
bulk-checkin.c
View File

@ -53,9 +53,10 @@ static void finish_bulk_checkin(struct bulk_checkin_state *state)
unlink(state->pack_tmp_name);
goto clear_exit;
} else if (state->nr_written == 1) {
finalize_hashfile(state->f, hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK,
CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
} else {
int fd = finalize_hashfile(state->f, hash, 0);
int fd = finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK, 0);
fixup_pack_header_footer(fd, hash, state->pack_tmp_name,
state->nr_written, hash,
state->offset);

48
cache.h
View File

@ -993,8 +993,54 @@ void reset_shared_repository(void);
extern int read_replace_refs;
extern char *git_replace_ref_base;
/*
* These values are used to help identify parts of a repository to fsync.
* FSYNC_COMPONENT_NONE identifies data that will not be a persistent part of the
* repository and so shouldn't be fsynced.
*/
enum fsync_component {
FSYNC_COMPONENT_NONE,
FSYNC_COMPONENT_LOOSE_OBJECT = 1 << 0,
FSYNC_COMPONENT_PACK = 1 << 1,
FSYNC_COMPONENT_PACK_METADATA = 1 << 2,
FSYNC_COMPONENT_COMMIT_GRAPH = 1 << 3,
FSYNC_COMPONENT_INDEX = 1 << 4,
};
#define FSYNC_COMPONENTS_OBJECTS (FSYNC_COMPONENT_LOOSE_OBJECT | \
FSYNC_COMPONENT_PACK)
#define FSYNC_COMPONENTS_DERIVED_METADATA (FSYNC_COMPONENT_PACK_METADATA | \
FSYNC_COMPONENT_COMMIT_GRAPH)
#define FSYNC_COMPONENTS_DEFAULT (FSYNC_COMPONENTS_OBJECTS | \
FSYNC_COMPONENTS_DERIVED_METADATA | \
~FSYNC_COMPONENT_LOOSE_OBJECT)
#define FSYNC_COMPONENTS_COMMITTED (FSYNC_COMPONENTS_OBJECTS)
#define FSYNC_COMPONENTS_ADDED (FSYNC_COMPONENTS_COMMITTED | \
FSYNC_COMPONENT_INDEX)
#define FSYNC_COMPONENTS_ALL (FSYNC_COMPONENT_LOOSE_OBJECT | \
FSYNC_COMPONENT_PACK | \
FSYNC_COMPONENT_PACK_METADATA | \
FSYNC_COMPONENT_COMMIT_GRAPH | \
FSYNC_COMPONENT_INDEX)
/*
* A bitmask indicating which components of the repo should be fsynced.
*/
extern enum fsync_component fsync_components;
extern int fsync_object_files;
extern int use_fsync;
enum fsync_method {
FSYNC_METHOD_FSYNC,
FSYNC_METHOD_WRITEOUT_ONLY
};
extern enum fsync_method fsync_method;
extern int core_preload_index;
extern int precomposed_unicode;
extern int protect_hfs;
@ -1715,6 +1761,8 @@ int copy_file_with_time(const char *dst, const char *src, int mode);
void write_or_die(int fd, const void *buf, size_t count);
void fsync_or_die(int fd, const char *);
int fsync_component(enum fsync_component component, int fd);
void fsync_component_or_die(enum fsync_component component, int fd, const char *msg);
ssize_t read_in_full(int fd, void *buf, size_t count);
ssize_t write_in_full(int fd, const void *buf, size_t count);

3
commit-graph.c
View File

@ -1952,7 +1952,8 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
}
close_commit_graph(ctx->r->objects);
finalize_hashfile(f, file_hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
finalize_hashfile(f, file_hash, FSYNC_COMPONENT_COMMIT_GRAPH,
CSUM_HASH_IN_STREAM | CSUM_FSYNC);
free_chunkfile(cf);
if (ctx->split) {

3
compat/mingw.h
View File

@ -329,6 +329,9 @@ int mingw_getpagesize(void);
#define getpagesize mingw_getpagesize
#endif
int win32_fsync_no_flush(int fd);
#define fsync_no_flush win32_fsync_no_flush
struct rlimit {
unsigned int rlim_cur;
};

28
compat/win32/flush.c Normal file
View File

@ -0,0 +1,28 @@
#include "git-compat-util.h"
#include <winternl.h>
#include "lazyload.h"
int win32_fsync_no_flush(int fd)
{
IO_STATUS_BLOCK io_status;
#define FLUSH_FLAGS_FILE_DATA_ONLY 1
DECLARE_PROC_ADDR(ntdll.dll, NTSTATUS, NTAPI, NtFlushBuffersFileEx,
HANDLE FileHandle, ULONG Flags, PVOID Parameters, ULONG ParameterSize,
PIO_STATUS_BLOCK IoStatusBlock);
if (!INIT_PROC_ADDR(NtFlushBuffersFileEx)) {
errno = ENOSYS;
return -1;
}
memset(&io_status, 0, sizeof(io_status));
if (NtFlushBuffersFileEx((HANDLE)_get_osfhandle(fd), FLUSH_FLAGS_FILE_DATA_ONLY,
NULL, 0, &io_status)) {
errno = EINVAL;
return -1;
}
return 0;
}

5
compat/winansi.c
View File

@ -4,11 +4,6 @@
#undef NOGDI
/*
* Including the appropriate header file for RtlGenRandom causes MSVC to see a
* redefinition of types in an incompatible way when including headers below.
*/
#undef HAVE_RTLGENRANDOM
#include "../git-compat-util.h"
#include <wingdi.h>
#include <winreg.h>

94
config.c
View File

@ -1323,6 +1323,79 @@ static int git_parse_maybe_bool_text(const char *value)
return -1;
}
static const struct fsync_component_name {
const char *name;
enum fsync_component component_bits;
} fsync_component_names[] = {
{ "loose-object", FSYNC_COMPONENT_LOOSE_OBJECT },
{ "pack", FSYNC_COMPONENT_PACK },
{ "pack-metadata", FSYNC_COMPONENT_PACK_METADATA },
{ "commit-graph", FSYNC_COMPONENT_COMMIT_GRAPH },
{ "index", FSYNC_COMPONENT_INDEX },
{ "objects", FSYNC_COMPONENTS_OBJECTS },
{ "derived-metadata", FSYNC_COMPONENTS_DERIVED_METADATA },
{ "committed", FSYNC_COMPONENTS_COMMITTED },
{ "added", FSYNC_COMPONENTS_ADDED },
{ "all", FSYNC_COMPONENTS_ALL },
};
static enum fsync_component parse_fsync_components(const char *var, const char *string)
{
enum fsync_component current = FSYNC_COMPONENTS_DEFAULT;
enum fsync_component positive = 0, negative = 0;
while (string) {
int i;
size_t len;
const char *ep;
int negated = 0;
int found = 0;
string = string + strspn(string, ", \t\n\r");
ep = strchrnul(string, ',');
len = ep - string;
if (!strcmp(string, "none")) {
current = FSYNC_COMPONENT_NONE;
goto next_name;
}
if (*string == '-') {
negated = 1;
string++;
len--;
if (!len)
warning(_("invalid value for variable %s"), var);
}
if (!len)
break;
for (i = 0; i < ARRAY_SIZE(fsync_component_names); ++i) {
const struct fsync_component_name *n = &fsync_component_names[i];
if (strncmp(n->name, string, len))
continue;
found = 1;
if (negated)
negative |= n->component_bits;
else
positive |= n->component_bits;
}
if (!found) {
char *component = xstrndup(string, len);
warning(_("ignoring unknown core.fsync component '%s'"), component);
free(component);
}
next_name:
string = ep;
}
return (current & ~negative) | positive;
}
int git_parse_maybe_bool(const char *value)
{
int v = git_parse_maybe_bool_text(value);
@ -1600,7 +1673,28 @@ static int git_default_core_config(const char *var, const char *value, void *cb)
return 0;
}
if (!strcmp(var, "core.fsync")) {
if (!value)
return config_error_nonbool(var);
fsync_components = parse_fsync_components(var, value);
return 0;
}
if (!strcmp(var, "core.fsyncmethod")) {
if (!value)
return config_error_nonbool(var);
if (!strcmp(value, "fsync"))
fsync_method = FSYNC_METHOD_FSYNC;
else if (!strcmp(value, "writeout-only"))
fsync_method = FSYNC_METHOD_WRITEOUT_ONLY;
else
warning(_("ignoring unknown core.fsyncMethod value '%s'"), value);
}
if (!strcmp(var, "core.fsyncobjectfiles")) {
if (fsync_object_files < 0)
warning(_("core.fsyncobjectfiles is deprecated; use core.fsync instead"));
fsync_object_files = git_config_bool(var, value);
return 0;
}

3
config.mak.uname
View File

@ -57,6 +57,7 @@ ifeq ($(uname_S),Linux)
HAVE_CLOCK_MONOTONIC = YesPlease
# -lrt is needed for clock_gettime on glibc <= 2.16
NEEDS_LIBRT = YesPlease
HAVE_SYNC_FILE_RANGE = YesPlease
HAVE_GETDELIM = YesPlease
FREAD_READS_DIRECTORIES = UnfortunatelyYes
BASIC_CFLAGS += -DHAVE_SYSINFO
@ -463,6 +464,7 @@ endif
CFLAGS =
BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE
COMPAT_OBJS = compat/msvc.o compat/winansi.o \
compat/win32/flush.o \
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/trace2_win32_process_info.o \
@ -640,6 +642,7 @@ ifeq ($(uname_S),MINGW)
COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\"
COMPAT_OBJS += compat/mingw.o compat/winansi.o \
compat/win32/trace2_win32_process_info.o \
compat/win32/flush.o \
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/dirent.o

8
configure.ac
View File

@ -1082,6 +1082,14 @@ AC_COMPILE_IFELSE([CLOCK_MONOTONIC_SRC],
[AC_MSG_RESULT([no])
HAVE_CLOCK_MONOTONIC=])
GIT_CONF_SUBST([HAVE_CLOCK_MONOTONIC])
#
# Define HAVE_SYNC_FILE_RANGE=YesPlease if sync_file_range is available.
GIT_CHECK_FUNC(sync_file_range,
[HAVE_SYNC_FILE_RANGE=YesPlease],
[HAVE_SYNC_FILE_RANGE])
GIT_CONF_SUBST([HAVE_SYNC_FILE_RANGE])
#
# Define NO_SETITIMER if you don't have setitimer.
GIT_CHECK_FUNC(setitimer,

16
contrib/buildsystems/CMakeLists.txt
View File

@ -261,10 +261,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
NOGDI OBJECT_CREATION_MODE=1 __USE_MINGW_ANSI_STDIO=0
USE_NED_ALLOCATOR OVERRIDE_STRDUP MMAP_PREVENTS_DELETE USE_WIN32_MMAP
UNICODE _UNICODE HAVE_WPGMPTR ENSURE_MSYSTEM_IS_SET HAVE_RTLGENRANDOM)
list(APPEND compat_SOURCES compat/mingw.c compat/winansi.c compat/win32/path-utils.c
compat/win32/pthread.c compat/win32mmap.c compat/win32/syslog.c
compat/win32/trace2_win32_process_info.c compat/win32/dirent.c
compat/nedmalloc/nedmalloc.c compat/strdup.c)
list(APPEND compat_SOURCES
compat/mingw.c
compat/winansi.c
compat/win32/flush.c
compat/win32/path-utils.c
compat/win32/pthread.c
compat/win32mmap.c
compat/win32/syslog.c
compat/win32/trace2_win32_process_info.c
compat/win32/dirent.c
compat/nedmalloc/nedmalloc.c
compat/strdup.c)
set(NO_UNIX_SOCKETS 1)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")

5
csum-file.c
View File

@ -58,7 +58,8 @@ static void free_hashfile(struct hashfile *f)
free(f);
}
int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int flags)
int finalize_hashfile(struct hashfile *f, unsigned char *result,
enum fsync_component component, unsigned int flags)
{
int fd;
@ -69,7 +70,7 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int fl
if (flags & CSUM_HASH_IN_STREAM)
flush(f, f->buffer, the_hash_algo->rawsz);
if (flags & CSUM_FSYNC)
fsync_or_die(f->fd, f->name);
fsync_component_or_die(component, f->fd, f->name);
if (flags & CSUM_CLOSE) {
if (close(f->fd))
die_errno("%s: sha1 file error on close", f->name);

3
csum-file.h
View File

@ -1,6 +1,7 @@
#ifndef CSUM_FILE_H
#define CSUM_FILE_H
#include "cache.h"
#include "hash.h"
struct progress;
@ -38,7 +39,7 @@ int hashfile_truncate(struct hashfile *, struct hashfile_checkpoint *);
struct hashfile *hashfd(int fd, const char *name);
struct hashfile *hashfd_check(const char *name);
struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp);
int finalize_hashfile(struct hashfile *, unsigned char *, unsigned int);
int finalize_hashfile(struct hashfile *, unsigned char *, enum fsync_component, unsigned int);
void hashwrite(struct hashfile *, const void *, unsigned int);
void hashflush(struct hashfile *f);
void crc32_begin(struct hashfile *);

4
environment.c
View File

@ -42,8 +42,10 @@ const char *git_attributes_file;
const char *git_hooks_path;
int zlib_compression_level = Z_BEST_SPEED;
int pack_compression_level = Z_DEFAULT_COMPRESSION;
int fsync_object_files;
int fsync_object_files = -1;
int use_fsync = -1;
enum fsync_method fsync_method = FSYNC_METHOD_DEFAULT;
enum fsync_component fsync_components = FSYNC_COMPONENTS_DEFAULT;
size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
size_t delta_base_cache_limit = 96 * 1024 * 1024;

30
git-compat-util.h
View File

@ -197,12 +197,6 @@
#endif
#include <windows.h>
#define GIT_WINDOWS_NATIVE
#ifdef HAVE_RTLGENRANDOM
/* This is required to get access to RtlGenRandom. */
#define SystemFunction036 NTAPI SystemFunction036
#include <NTSecAPI.h>
#undef SystemFunction036
#endif
#endif
#include <unistd.h>
@ -1263,6 +1257,30 @@ __attribute__((format (printf, 3, 4))) NORETURN
void BUG_fl(const char *file, int line, const char *fmt, ...);
#define BUG(...) BUG_fl(__FILE__, __LINE__, __VA_ARGS__)
#ifdef __APPLE__
#define FSYNC_METHOD_DEFAULT FSYNC_METHOD_WRITEOUT_ONLY
#else
#define FSYNC_METHOD_DEFAULT FSYNC_METHOD_FSYNC
#endif
enum fsync_action {
FSYNC_WRITEOUT_ONLY,
FSYNC_HARDWARE_FLUSH
};
/*
* Issues an fsync against the specified file according to the specified mode.
*
* FSYNC_WRITEOUT_ONLY attempts to use interfaces available on some operating
* systems to flush the OS cache without issuing a flush command to the storage
* controller. If those interfaces are unavailable, the function fails with
* ENOSYS.
*
* FSYNC_HARDWARE_FLUSH does an OS writeout and hardware flush to ensure that
* changes are durable. It is not expected to fail.
*/
int git_fsync(int fd, enum fsync_action action);
/*
* Preserves errno, prints a message, but gives no warning for ENOENT.
* Returns 0 on success, which includes trying to unlink an object that does

3
midx.c
View File

@ -1438,7 +1438,8 @@ static int write_midx_internal(const char *object_dir,
write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
write_chunkfile(cf, &ctx);
finalize_hashfile(f, midx_hash, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
finalize_hashfile(f, midx_hash, FSYNC_COMPONENT_PACK_METADATA,
CSUM_FSYNC | CSUM_HASH_IN_STREAM);
free_chunkfile(cf);
if (flags & MIDX_WRITE_REV_INDEX &&

13
object-file.c
View File

@ -1889,11 +1889,16 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
/* Finalize a file on disk, and close it. */
static void close_loose_object(int fd)
{
if (!the_repository->objects->odb->will_destroy) {
if (fsync_object_files)
fsync_or_die(fd, "loose object file");
}
if (the_repository->objects->odb->will_destroy)
goto out;
if (fsync_object_files > 0)
fsync_or_die(fd, "loose object file");
else
fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
"loose object file");
out:
if (close(fd) != 0)
die_errno(_("error when closing loose object file"));
}

3
pack-bitmap-write.c
View File

@ -719,7 +719,8 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
if (options & BITMAP_OPT_HASH_CACHE)
write_hash_cache(f, index, index_nr);
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
finalize_hashfile(f, NULL, FSYNC_COMPONENT_PACK_METADATA,
CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
if (adjust_shared_perm(tmp_file.buf))
die_errno("unable to make temporary bitmap file readable");

13
pack-write.c
View File

@ -159,9 +159,9 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec
}
hashwrite(f, sha1, the_hash_algo->rawsz);
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE |
((opts->flags & WRITE_IDX_VERIFY)
? 0 : CSUM_FSYNC));
finalize_hashfile(f, NULL, FSYNC_COMPONENT_PACK_METADATA,
CSUM_HASH_IN_STREAM | CSUM_CLOSE |
((opts->flags & WRITE_IDX_VERIFY) ? 0 : CSUM_FSYNC));
return index_name;
}
@ -281,8 +281,9 @@ const char *write_rev_file_order(const char *rev_name,
if (rev_name && adjust_shared_perm(rev_name) < 0)
die(_("failed to make %s readable"), rev_name);
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE |
((flags & WRITE_IDX_VERIFY) ? 0 : CSUM_FSYNC));
finalize_hashfile(f, NULL, FSYNC_COMPONENT_PACK_METADATA,
CSUM_HASH_IN_STREAM | CSUM_CLOSE |
((flags & WRITE_IDX_VERIFY) ? 0 : CSUM_FSYNC));
return rev_name;
}
@ -390,7 +391,7 @@ void fixup_pack_header_footer(int pack_fd,
the_hash_algo->final_fn(partial_pack_hash, &old_hash_ctx);
the_hash_algo->final_fn(new_pack_hash, &new_hash_ctx);
write_or_die(pack_fd, new_pack_hash, the_hash_algo->rawsz);
fsync_or_die(pack_fd, pack_name);
fsync_component_or_die(FSYNC_COMPONENT_PACK, pack_fd, pack_name);
}
char *index_pack_lockfile(int ip_out, int *is_well_formed)

19
read-cache.c
View File

@ -2842,7 +2842,7 @@ static int record_ieot(void)
* rely on it.
*/
static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
int strip_extensions)
int strip_extensions, unsigned flags)
{
uint64_t start = getnanotime();
struct hashfile *f;
@ -2856,6 +2856,7 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
int drop_cache_tree = istate->drop_cache_tree;
off_t offset;
int csum_fsync_flag;
int ieot_entries = 1;
struct index_entry_offset_table *ieot = NULL;
int nr, nr_threads;
@ -3089,7 +3090,13 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
return -1;
}
finalize_hashfile(f, istate->oid.hash, CSUM_HASH_IN_STREAM);
csum_fsync_flag = 0;
if (!alternate_index_output && (flags & COMMIT_LOCK))
csum_fsync_flag = CSUM_FSYNC;
finalize_hashfile(f, istate->oid.hash, FSYNC_COMPONENT_INDEX,
CSUM_HASH_IN_STREAM | csum_fsync_flag);
if (close_tempfile_gently(tempfile)) {
error(_("could not close '%s'"), get_tempfile_path(tempfile));
return -1;
@ -3144,7 +3151,7 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l
*/
trace2_region_enter_printf("index", "do_write_index", the_repository,
"%s", get_lock_file_path(lock));
ret = do_write_index(istate, lock->tempfile, 0);
ret = do_write_index(istate, lock->tempfile, 0, flags);
trace2_region_leave_printf("index", "do_write_index", the_repository,
"%s", get_lock_file_path(lock));
@ -3238,7 +3245,7 @@ static int clean_shared_index_files(const char *current_hex)
}
static int write_shared_index(struct index_state *istate,
struct tempfile **temp)
struct tempfile **temp, unsigned flags)
{
struct split_index *si = istate->split_index;
int ret, was_full = !istate->sparse_index;
@ -3248,7 +3255,7 @@ static int write_shared_index(struct index_state *istate,
trace2_region_enter_printf("index", "shared/do_write_index",
the_repository, "%s", get_tempfile_path(*temp));
ret = do_write_index(si->base, *temp, 1);
ret = do_write_index(si->base, *temp, 1, flags);
trace2_region_leave_printf("index", "shared/do_write_index",
the_repository, "%s", get_tempfile_path(*temp));
@ -3357,7 +3364,7 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock,
ret = do_write_locked_index(istate, lock, flags);
goto out;
}
ret = write_shared_index(istate, &temp);
ret = write_shared_index(istate, &temp, flags);
saved_errno = errno;
if (is_tempfile_active(temp))

71
wrapper.c
View File

@ -4,6 +4,13 @@
#include "cache.h"
#include "config.h"
#ifdef HAVE_RTLGENRANDOM
/* This is required to get access to RtlGenRandom. */
#define SystemFunction036 NTAPI SystemFunction036
#include <NTSecAPI.h>
#undef SystemFunction036
#endif
static int memory_limit_check(size_t size, int gentle)
{
static size_t limit = 0;
@ -539,6 +546,70 @@ int xmkstemp_mode(char *filename_template, int mode)
return fd;
}
/*
* Some platforms return EINTR from fsync. Since fsync is invoked in some
* cases by a wrapper that dies on failure, do not expose EINTR to callers.
*/
static int fsync_loop(int fd)
{
int err;
do {
err = fsync(fd);
} while (err < 0 && errno == EINTR);
return err;
}
int git_fsync(int fd, enum fsync_action action)
{
switch (action) {
case FSYNC_WRITEOUT_ONLY:
#ifdef __APPLE__
/*
* On macOS, fsync just causes filesystem cache writeback but
* does not flush hardware caches.
*/
return fsync_loop(fd);
#endif
#ifdef HAVE_SYNC_FILE_RANGE
/*
* On linux 2.6.17 and above, sync_file_range is the way to
* issue a writeback without a hardware flush. An offset of
* 0 and size of 0 indicates writeout of the entire file and the
* wait flags ensure that all dirty data is written to the disk
* (potentially in a disk-side cache) before we continue.
*/
return sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WAIT_BEFORE |
SYNC_FILE_RANGE_WRITE |
SYNC_FILE_RANGE_WAIT_AFTER);
#endif
#ifdef fsync_no_flush
return fsync_no_flush(fd);
#endif
errno = ENOSYS;
return -1;
case FSYNC_HARDWARE_FLUSH:
/*
* On macOS, a special fcntl is required to really flush the
* caches within the storage controller. As of this writing,
* this is a very expensive operation on Apple SSDs.
*/
#ifdef __APPLE__
return fcntl(fd, F_FULLFSYNC);
#else
return fsync_loop(fd);
#endif
default:
BUG("unexpected git_fsync(%d) call", action);
}
}
static int warn_if_unremovable(const char *op, const char *file, int rc)
{
int err;

33
write-or-die.c
View File

@ -56,16 +56,37 @@ void fprintf_or_die(FILE *f, const char *fmt, ...)
}
}
void fsync_or_die(int fd, const char *msg)
static int maybe_fsync(int fd)
{
if (use_fsync < 0)
use_fsync = git_env_bool("GIT_TEST_FSYNC", 1);
if (!use_fsync)
return;
while (fsync(fd) < 0) {
if (errno != EINTR)
die_errno("fsync error on '%s'", msg);
}
return 0;
if (fsync_method == FSYNC_METHOD_WRITEOUT_ONLY &&
git_fsync(fd, FSYNC_WRITEOUT_ONLY) >= 0)
return 0;
return git_fsync(fd, FSYNC_HARDWARE_FLUSH);
}
void fsync_or_die(int fd, const char *msg)
{
if (maybe_fsync(fd) < 0)
die_errno("fsync error on '%s'", msg);
}
int fsync_component(enum fsync_component component, int fd)
{
if (fsync_components & component)
return maybe_fsync(fd);
return 0;
}
void fsync_component_or_die(enum fsync_component component, int fd, const char *msg)
{
if (fsync_components & component)
fsync_or_die(fd, msg);
}
void write_or_die(int fd, const void *buf, size_t count)