undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'bp/sub-process-convert-filter'

Browse Source 
Code from "conversion using external process" codepath has been
extracted to a separate sub-process.[ch] module.

* bp/sub-process-convert-filter:
  convert: update subprocess_read_status() to not die on EOF
  sub-process: move sub-process functions into separate files
  convert: rename reusable sub-process functions
  convert: update generic functions to only use generic data structures
  convert: separate generic structures and variables from the filter specific ones
  convert: split start_multi_file_filter() into two separate functions
  pkt-line: annotate packet_writel with LAST_ARG_MUST_BE_NULL
  convert: move packet_write_line() into pkt-line as packet_writel()
  pkt-line: add packet_read_line_gently()
  pkt-line: fix packet_read_line() to handle len < 0 errors
  convert: remove erroneous tests for errno == EPIPE
This commit is contained in:
Junio C Hamano 2017-05-30 11:16:42 +09:00
commit ae7785de0e
7 changed files with 293 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
Documentation/technical/api-sub-process.txt Normal file
View File

@ -0,0 +1,59 @@
sub-process API
===============
The sub-process API makes it possible to run background sub-processes
for the entire lifetime of a Git invocation. If Git needs to communicate
with an external process multiple times, then this can reduces the process
invocation overhead. Git and the sub-process communicate through stdin and
stdout.
The sub-processes are kept in a hashmap by command name and looked up
via the subprocess_find_entry function. If an existing instance can not
be found then a new process should be created and started. When the
parent git command terminates, all sub-processes are also terminated.
This API is based on the run-command API.
Data structures
---------------
* `struct subprocess_entry`
The sub-process structure. Members should not be accessed directly.
Types
-----
'int(*subprocess_start_fn)(struct subprocess_entry *entry)'::
User-supplied function to initialize the sub-process. This is
typically used to negotiate the interface version and capabilities.
Functions
---------
`cmd2process_cmp`::
Function to test two subprocess hashmap entries for equality.
`subprocess_start`::
Start a subprocess and add it to the subprocess hashmap.
`subprocess_stop`::
Kill a subprocess and remove it from the subprocess hashmap.
`subprocess_find_entry`::
Find a subprocess in the subprocess hashmap.
`subprocess_get_child_process`::
Get the underlying `struct child_process` from a subprocess.
`subprocess_read_status`::
Helper function to read packets looking for the last "status=<foo>"
key/value pair.

1
Makefile
View File

@ -842,6 +842,7 @@ LIB_OBJS += streaming.o
LIB_OBJS += string-list.o
LIB_OBJS += submodule.o
LIB_OBJS += submodule-config.o
LIB_OBJS += sub-process.o
LIB_OBJS += symlinks.o
LIB_OBJS += tag.o
LIB_OBJS += tempfile.o

161
convert.c
View File

@ -4,6 +4,7 @@
#include "quote.h"
#include "sigchain.h"
#include "pkt-line.h"
#include "sub-process.h"
/*
* convert.c - convert a file when checking it out and checking it in.
@ -497,126 +498,26 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le
#define CAP_SMUDGE (1u<<1)
struct cmd2process {
struct hashmap_entry ent; /* must be the first member! */
struct subprocess_entry subprocess; /* must be the first member! */
unsigned int supported_capabilities;
const char *cmd;
struct child_process process;
};
static int cmd_process_map_initialized;
static struct hashmap cmd_process_map;
static int subprocess_map_initialized;
static struct hashmap subprocess_map;
static int cmd2process_cmp(const struct cmd2process *e1,
const struct cmd2process *e2,
const void *unused)
{
return strcmp(e1->cmd, e2->cmd);
}
static struct cmd2process *find_multi_file_filter_entry(struct hashmap *hashmap, const char *cmd)
{
struct cmd2process key;
hashmap_entry_init(&key, strhash(cmd));
key.cmd = cmd;
return hashmap_get(hashmap, &key, NULL);
}
static int packet_write_list(int fd, const char *line, ...)
{
va_list args;
int err;
va_start(args, line);
for (;;) {
if (!line)
break;
if (strlen(line) > LARGE_PACKET_DATA_MAX)
return -1;
err = packet_write_fmt_gently(fd, "%s\n", line);
if (err)
return err;
line = va_arg(args, const char*);
}
va_end(args);
return packet_flush_gently(fd);
}
static void read_multi_file_filter_status(int fd, struct strbuf *status)
{
struct strbuf **pair;
char *line;
for (;;) {
line = packet_read_line(fd, NULL);
if (!line)
break;
pair = strbuf_split_str(line, '=', 2);
if (pair[0] && pair[0]->len && pair[1]) {
/* the last "status=<foo>" line wins */
if (!strcmp(pair[0]->buf, "status=")) {
strbuf_reset(status);
strbuf_addbuf(status, pair[1]);
}
}
strbuf_list_free(pair);
}
}
static void kill_multi_file_filter(struct hashmap *hashmap, struct cmd2process *entry)
{
if (!entry)
return;
entry->process.clean_on_exit = 0;
kill(entry->process.pid, SIGTERM);
finish_command(&entry->process);
hashmap_remove(hashmap, entry, NULL);
free(entry);
}
static void stop_multi_file_filter(struct child_process *process)
{
sigchain_push(SIGPIPE, SIG_IGN);
/* Closing the pipe signals the filter to initiate a shutdown. */
close(process->in);
close(process->out);
sigchain_pop(SIGPIPE);
/* Finish command will wait until the shutdown is complete. */
finish_command(process);
}
static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, const char *cmd)
static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
{
int err;
struct cmd2process *entry;
struct child_process *process;
const char *argv[] = { cmd, NULL };
struct cmd2process *entry = (struct cmd2process *)subprocess;
struct string_list cap_list = STRING_LIST_INIT_NODUP;
char *cap_buf;
const char *cap_name;
entry = xmalloc(sizeof(*entry));
entry->cmd = cmd;
entry->supported_capabilities = 0;
process = &entry->process;
child_process_init(process);
process->argv = argv;
process->use_shell = 1;
process->in = -1;
process->out = -1;
process->clean_on_exit = 1;
process->clean_on_exit_handler = stop_multi_file_filter;
if (start_command(process)) {
error("cannot fork to run external filter '%s'", cmd);
return NULL;
}
hashmap_entry_init(entry, strhash(cmd));
struct child_process *process = &subprocess->process;
const char *cmd = subprocess->cmd;
sigchain_push(SIGPIPE, SIG_IGN);
err = packet_write_list(process->in, "git-filter-client", "version=2", NULL);
err = packet_writel(process->in, "git-filter-client", "version=2", NULL);
if (err)
goto done;
@ -632,7 +533,7 @@ static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, cons
if (err)
goto done;
err = packet_write_list(process->in, "capability=clean", "capability=smudge", NULL);
err = packet_writel(process->in, "capability=clean", "capability=smudge", NULL);
for (;;) {
cap_buf = packet_read_line(process->out, NULL);
@ -661,14 +562,7 @@ static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, cons
done:
sigchain_pop(SIGPIPE);
if (err || errno == EPIPE) {
error("initialization for external filter '%s' failed", cmd);
kill_multi_file_filter(hashmap, entry);
return NULL;
}
hashmap_add(hashmap, entry);
return entry;
return err;
}
static int apply_multi_file_filter(const char *path, const char *src, size_t len,
@ -682,22 +576,26 @@ static int apply_multi_file_filter(const char *path, const char *src, size_t len
struct strbuf filter_status = STRBUF_INIT;
const char *filter_type;
if (!cmd_process_map_initialized) {
cmd_process_map_initialized = 1;
hashmap_init(&cmd_process_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
if (!subprocess_map_initialized) {
subprocess_map_initialized = 1;
hashmap_init(&subprocess_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
entry = NULL;
} else {
entry = find_multi_file_filter_entry(&cmd_process_map, cmd);
entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
}
fflush(NULL);
if (!entry) {
entry = start_multi_file_filter(&cmd_process_map, cmd);
if (!entry)
entry = xmalloc(sizeof(*entry));
entry->supported_capabilities = 0;
if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
free(entry);
return 0;
}
}
process = &entry->process;
process = &entry->subprocess.process;
if (!(wanted_capability & entry->supported_capabilities))
return 0;
@ -737,7 +635,10 @@ static int apply_multi_file_filter(const char *path, const char *src, size_t len
if (err)
goto done;
read_multi_file_filter_status(process->out, &filter_status);
err = subprocess_read_status(process->out, &filter_status);
if (err)
goto done;
err = strcmp(filter_status.buf, "success");
if (err)
goto done;
@ -746,13 +647,16 @@ static int apply_multi_file_filter(const char *path, const char *src, size_t len
if (err)
goto done;
read_multi_file_filter_status(process->out, &filter_status);
err = subprocess_read_status(process->out, &filter_status);
if (err)
goto done;
err = strcmp(filter_status.buf, "success");
done:
sigchain_pop(SIGPIPE);
if (err || errno == EPIPE) {
if (err) {
if (!strcmp(filter_status.buf, "error")) {
/* The filter signaled a problem with the file. */
} else if (!strcmp(filter_status.buf, "abort")) {
@ -768,7 +672,8 @@ done:
* Force shutdown and restart if another blob requires filtering.
*/
error("external filter '%s' failed", cmd);
kill_multi_file_filter(&cmd_process_map, entry);
subprocess_stop(&subprocess_map, &entry->subprocess);
free(entry);
}
} else {
strbuf_swap(dst, &nbuf);

33
pkt-line.c
View File

@ -171,6 +171,25 @@ int packet_write_fmt_gently(int fd, const char *fmt, ...)
return status;
}
int packet_writel(int fd, const char *line, ...)
{
va_list args;
int err;
va_start(args, line);
for (;;) {
if (!line)
break;
if (strlen(line) > LARGE_PACKET_DATA_MAX)
return -1;
err = packet_write_fmt_gently(fd, "%s\n", line);
if (err)
return err;
line = va_arg(args, const char*);
}
va_end(args);
return packet_flush_gently(fd);
}
static int packet_write_gently(const int fd_out, const char *buf, size_t size)
{
static char packet_write_buffer[LARGE_PACKET_MAX];
@ -315,7 +334,7 @@ static char *packet_read_line_generic(int fd,
PACKET_READ_CHOMP_NEWLINE);
if (dst_len)
*dst_len = len;
return len ? packet_buffer : NULL;
return (len > 0) ? packet_buffer : NULL;
}
char *packet_read_line(int fd, int *len_p)
@ -323,6 +342,18 @@ char *packet_read_line(int fd, int *len_p)
return packet_read_line_generic(fd, NULL, NULL, len_p);
}
int packet_read_line_gently(int fd, int *dst_len, char **dst_line)
{
int len = packet_read(fd, NULL, NULL,
packet_buffer, sizeof(packet_buffer),
PACKET_READ_CHOMP_NEWLINE|PACKET_READ_GENTLE_ON_EOF);
if (dst_len)
*dst_len = len;
if (dst_line)
*dst_line = (len > 0) ? packet_buffer : NULL;
return len;
}
char *packet_read_line_buf(char **src, size_t *src_len, int *dst_len)
{
return packet_read_line_generic(-1, src, src_len, dst_len);

13
pkt-line.h
View File

@ -25,6 +25,8 @@ void packet_buf_flush(struct strbuf *buf);
void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
int packet_flush_gently(int fd);
int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
LAST_ARG_MUST_BE_NULL
int packet_writel(int fd, const char *line, ...);
int write_packetized_from_fd(int fd_in, int fd_out);
int write_packetized_from_buf(const char *src_in, size_t len, int fd_out);
@ -73,6 +75,17 @@ int packet_read(int fd, char **src_buffer, size_t *src_len, char
*/
char *packet_read_line(int fd, int *size);
/*
* Convenience wrapper for packet_read that sets the PACKET_READ_GENTLE_ON_EOF
* and CHOMP_NEWLINE options. The return value specifies the number of bytes
* read into the buffer or -1 on truncated input. If the *dst_line parameter
* is not NULL it will return NULL for a flush packet or when the number of
* bytes copied is zero and otherwise points to a static buffer (that may be
* overwritten by subsequent calls). If the size parameter is not NULL, the
* length of the packet is written to it.
*/
int packet_read_line_gently(int fd, int *size, char **dst_line);
/*
* Same as packet_read_line, but read from a buf rather than a descriptor;
* see packet_read for details on how src_* is used.

106
sub-process.c Normal file
View File

@ -0,0 +1,106 @@
/*
* Generic implementation of background process infrastructure.
*/
#include "sub-process.h"
#include "sigchain.h"
#include "pkt-line.h"
int cmd2process_cmp(const struct subprocess_entry *e1,
const struct subprocess_entry *e2,
const void *unused)
{
return strcmp(e1->cmd, e2->cmd);
}
struct subprocess_entry *subprocess_find_entry(struct hashmap *hashmap, const char *cmd)
{
struct subprocess_entry key;
hashmap_entry_init(&key, strhash(cmd));
key.cmd = cmd;
return hashmap_get(hashmap, &key, NULL);
}
int subprocess_read_status(int fd, struct strbuf *status)
{
struct strbuf **pair;
char *line;
int len;
for (;;) {
len = packet_read_line_gently(fd, NULL, &line);
if ((len < 0) || !line)
break;
pair = strbuf_split_str(line, '=', 2);
if (pair[0] && pair[0]->len && pair[1]) {
/* the last "status=<foo>" line wins */
if (!strcmp(pair[0]->buf, "status=")) {
strbuf_reset(status);
strbuf_addbuf(status, pair[1]);
}
}
strbuf_list_free(pair);
}
return (len < 0) ? len : 0;
}
void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry)
{
if (!entry)
return;
entry->process.clean_on_exit = 0;
kill(entry->process.pid, SIGTERM);
finish_command(&entry->process);
hashmap_remove(hashmap, entry, NULL);
}
static void subprocess_exit_handler(struct child_process *process)
{
sigchain_push(SIGPIPE, SIG_IGN);
/* Closing the pipe signals the subprocess to initiate a shutdown. */
close(process->in);
close(process->out);
sigchain_pop(SIGPIPE);
/* Finish command will wait until the shutdown is complete. */
finish_command(process);
}
int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd,
subprocess_start_fn startfn)
{
int err;
struct child_process *process;
const char *argv[] = { cmd, NULL };
entry->cmd = cmd;
process = &entry->process;
child_process_init(process);
process->argv = argv;
process->use_shell = 1;
process->in = -1;
process->out = -1;
process->clean_on_exit = 1;
process->clean_on_exit_handler = subprocess_exit_handler;
err = start_command(process);
if (err) {
error("cannot fork to run subprocess '%s'", cmd);
return err;
}
hashmap_entry_init(entry, strhash(cmd));
err = startfn(entry);
if (err) {
error("initialization for subprocess '%s' failed", cmd);
subprocess_stop(hashmap, entry);
return err;
}
hashmap_add(hashmap, entry);
return 0;
}

49
sub-process.h Normal file
View File

@ -0,0 +1,49 @@
#ifndef SUBPROCESS_H
#define SUBPROCESS_H
#include "git-compat-util.h"
#include "hashmap.h"
#include "run-command.h"
/*
* Generic implementation of background process infrastructure.
* See Documentation/technical/api-background-process.txt.
*/
/* data structures */
struct subprocess_entry {
struct hashmap_entry ent; /* must be the first member! */
const char *cmd;
struct child_process process;
};
/* subprocess functions */
int cmd2process_cmp(const struct subprocess_entry *e1,
const struct subprocess_entry *e2, const void *unused);
typedef int(*subprocess_start_fn)(struct subprocess_entry *entry);
int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd,
subprocess_start_fn startfn);
void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry);
struct subprocess_entry *subprocess_find_entry(struct hashmap *hashmap, const char *cmd);
/* subprocess helper functions */
static inline struct child_process *subprocess_get_child_process(
struct subprocess_entry *entry)
{
return &entry->process;
}
/*
* Helper function that will read packets looking for "status=<foo>"
* key/value pairs and return the value from the last "status" packet
*/
int subprocess_read_status(int fd, struct strbuf *status);
#endif