undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'jc/streaming-filter' into next

Browse Source 
* jc/streaming-filter:
  t0021: test application of both crlf and ident
  t0021-conversion.sh: fix NoTerminatingSymbolAtEOF test
  streaming: filter cascading
  streaming filter: ident filter
  Add LF-to-CRLF streaming conversion
  stream filter: add "no more input" to the filters
  Add streaming filter API
  convert.h: move declarations for conversion from cache.h
This commit is contained in:
Junio C Hamano 2011-06-29 17:09:28 -07:00
commit 9901923cf0
7 changed files with 620 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
cache.h
View File

@ -6,6 +6,7 @@
#include "hash.h"
#include "advice.h"
#include "gettext.h"
#include "convert.h"
#include SHA1_HEADER
#ifndef git_SHA_CTX
@ -582,35 +583,6 @@ extern int fsync_object_files;
extern int core_preload_index;
extern int core_apply_sparse_checkout;
enum safe_crlf {
SAFE_CRLF_FALSE = 0,
SAFE_CRLF_FAIL = 1,
SAFE_CRLF_WARN = 2
};
extern enum safe_crlf safe_crlf;
enum auto_crlf {
AUTO_CRLF_FALSE = 0,
AUTO_CRLF_TRUE = 1,
AUTO_CRLF_INPUT = -1
};
extern enum auto_crlf auto_crlf;
enum eol {
EOL_UNSET,
EOL_CRLF,
EOL_LF,
#ifdef NATIVE_CRLF
EOL_NATIVE = EOL_CRLF
#else
EOL_NATIVE = EOL_LF
#endif
};
extern enum eol core_eol;
enum branch_track {
BRANCH_TRACK_UNSPECIFIED = -1,
BRANCH_TRACK_NEVER = 0,
@ -1162,14 +1134,6 @@ extern void trace_strbuf(const char *key, const struct strbuf *buf);
void packet_trace_identity(const char *prog);
/* convert.c */
/* returns 1 if *dst was used */
extern int convert_to_git(const char *path, const char *src, size_t len,
struct strbuf *dst, enum safe_crlf checksafe);
extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst);
extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst);
extern int can_bypass_conversion(const char *path);
/* add */
/*
* return 0 if success, 1 - if addition of a file failed and

392
convert.c
View File

@ -814,25 +814,399 @@ int renormalize_buffer(const char *path, const char *src, size_t len, struct str
return ret | convert_to_git(path, src, len, dst, 0);
}
/*****************************************************************
*
* Streaming converison support
*
*****************************************************************/
typedef int (*filter_fn)(struct stream_filter *,
const char *input, size_t *isize_p,
char *output, size_t *osize_p);
typedef void (*free_fn)(struct stream_filter *);
struct stream_filter_vtbl {
filter_fn filter;
free_fn free;
};
struct stream_filter {
struct stream_filter_vtbl *vtbl;
};
static int null_filter_fn(struct stream_filter *filter,
const char *input, size_t *isize_p,
char *output, size_t *osize_p)
{
size_t count;
if (!input)
return 0; /* we do not keep any states */
count = *isize_p;
if (*osize_p < count)
count = *osize_p;
if (count) {
memmove(output, input, count);
*isize_p -= count;
*osize_p -= count;
}
return 0;
}
static void null_free_fn(struct stream_filter *filter)
{
; /* nothing -- null instances are shared */
}
static struct stream_filter_vtbl null_vtbl = {
null_filter_fn,
null_free_fn,
};
static struct stream_filter null_filter_singleton = {
&null_vtbl,
};
int is_null_stream_filter(struct stream_filter *filter)
{
return filter == &null_filter_singleton;
}
/*
* You would be crazy to set CRLF, smuge/clean or ident to
* a large binary blob you would want us not to slurp into
* the memory!
* LF-to-CRLF filter
*/
int can_bypass_conversion(const char *path)
static int lf_to_crlf_filter_fn(struct stream_filter *filter,
const char *input, size_t *isize_p,
char *output, size_t *osize_p)
{
size_t count;
if (!input)
return 0; /* we do not keep any states */
count = *isize_p;
if (count) {
size_t i, o;
for (i = o = 0; o < *osize_p && i < count; i++) {
char ch = input[i];
if (ch == '\n') {
if (o + 1 < *osize_p)
output[o++] = '\r';
else
break;
}
output[o++] = ch;
}
*osize_p -= o;
*isize_p -= i;
}
return 0;
}
static struct stream_filter_vtbl lf_to_crlf_vtbl = {
lf_to_crlf_filter_fn,
null_free_fn,
};
static struct stream_filter lf_to_crlf_filter_singleton = {
&lf_to_crlf_vtbl,
};
/*
* Cascade filter
*/
#define FILTER_BUFFER 1024
struct cascade_filter {
struct stream_filter filter;
struct stream_filter *one;
struct stream_filter *two;
char buf[FILTER_BUFFER];
int end, ptr;
};
static int cascade_filter_fn(struct stream_filter *filter,
const char *input, size_t *isize_p,
char *output, size_t *osize_p)
{
struct cascade_filter *cas = (struct cascade_filter *) filter;
size_t filled = 0;
size_t sz = *osize_p;
size_t to_feed, remaining;
/*
* input -- (one) --> buf -- (two) --> output
*/
while (filled < sz) {
remaining = sz - filled;
/* do we already have something to feed two with? */
if (cas->ptr < cas->end) {
to_feed = cas->end - cas->ptr;
if (stream_filter(cas->two,
cas->buf + cas->ptr, &to_feed,
output + filled, &remaining))
return -1;
cas->ptr += (cas->end - cas->ptr) - to_feed;
filled = sz - remaining;
continue;
}
/* feed one from upstream and have it emit into our buffer */
to_feed = input ? *isize_p : 0;
if (input && !to_feed)
break;
remaining = sizeof(cas->buf);
if (stream_filter(cas->one,
input, &to_feed,
cas->buf, &remaining))
return -1;
cas->end = sizeof(cas->buf) - remaining;
cas->ptr = 0;
if (input) {
size_t fed = *isize_p - to_feed;
*isize_p -= fed;
input += fed;
}
/* do we know that we drained one completely? */
if (input || cas->end)
continue;
/* tell two to drain; we have nothing more to give it */
to_feed = 0;
remaining = sz - filled;
if (stream_filter(cas->two,
NULL, &to_feed,
output + filled, &remaining))
return -1;
if (remaining == (sz - filled))
break; /* completely drained two */
filled = sz - remaining;
}
*osize_p -= filled;
return 0;
}
static void cascade_free_fn(struct stream_filter *filter)
{
struct cascade_filter *cas = (struct cascade_filter *)filter;
free_stream_filter(cas->one);
free_stream_filter(cas->two);
free(filter);
}
static struct stream_filter_vtbl cascade_vtbl = {
cascade_filter_fn,
cascade_free_fn,
};
static struct stream_filter *cascade_filter(struct stream_filter *one,
struct stream_filter *two)
{
struct cascade_filter *cascade;
if (!one || is_null_stream_filter(one))
return two;
if (!two || is_null_stream_filter(two))
return one;
cascade = xmalloc(sizeof(*cascade));
cascade->one = one;
cascade->two = two;
cascade->end = cascade->ptr = 0;
cascade->filter.vtbl = &cascade_vtbl;
return (struct stream_filter *)cascade;
}
/*
* ident filter
*/
#define IDENT_DRAINING (-1)
#define IDENT_SKIPPING (-2)
struct ident_filter {
struct stream_filter filter;
struct strbuf left;
int state;
char ident[45]; /* ": x40 $" */
};
static int is_foreign_ident(const char *str)
{
int i;
if (prefixcmp(str, "$Id: "))
return 0;
for (i = 5; str[i]; i++) {
if (isspace(str[i]) && str[i+1] != '$')
return 1;
}
return 0;
}
static void ident_drain(struct ident_filter *ident, char **output_p, size_t *osize_p)
{
size_t to_drain = ident->left.len;
if (*osize_p < to_drain)
to_drain = *osize_p;
if (to_drain) {
memcpy(*output_p, ident->left.buf, to_drain);
strbuf_remove(&ident->left, 0, to_drain);
*output_p += to_drain;
*osize_p -= to_drain;
}
if (!ident->left.len)
ident->state = 0;
}
static int ident_filter_fn(struct stream_filter *filter,
const char *input, size_t *isize_p,
char *output, size_t *osize_p)
{
struct ident_filter *ident = (struct ident_filter *)filter;
static const char head[] = "$Id";
if (!input) {
/* drain upon eof */
switch (ident->state) {
default:
strbuf_add(&ident->left, head, ident->state);
case IDENT_SKIPPING:
/* fallthru */
case IDENT_DRAINING:
ident_drain(ident, &output, osize_p);
}
return 0;
}
while (*isize_p || (ident->state == IDENT_DRAINING)) {
int ch;
if (ident->state == IDENT_DRAINING) {
ident_drain(ident, &output, osize_p);
if (!*osize_p)
break;
continue;
}
ch = *(input++);
(*isize_p)--;
if (ident->state == IDENT_SKIPPING) {
/*
* Skipping until '$' or LF, but keeping them
* in case it is a foreign ident.
*/
strbuf_addch(&ident->left, ch);
if (ch != '\n' && ch != '$')
continue;
if (ch == '$' && !is_foreign_ident(ident->left.buf)) {
strbuf_setlen(&ident->left, sizeof(head) - 1);
strbuf_addstr(&ident->left, ident->ident);
}
ident->state = IDENT_DRAINING;
continue;
}
if (ident->state < sizeof(head) &&
head[ident->state] == ch) {
ident->state++;
continue;
}
if (ident->state)
strbuf_add(&ident->left, head, ident->state);
if (ident->state == sizeof(head) - 1) {
if (ch != ':' && ch != '$') {
strbuf_addch(&ident->left, ch);
ident->state = 0;
continue;
}
if (ch == ':') {
strbuf_addch(&ident->left, ch);
ident->state = IDENT_SKIPPING;
} else {
strbuf_addstr(&ident->left, ident->ident);
ident->state = IDENT_DRAINING;
}
continue;
}
strbuf_addch(&ident->left, ch);
ident->state = IDENT_DRAINING;
}
return 0;
}
static void ident_free_fn(struct stream_filter *filter)
{
struct ident_filter *ident = (struct ident_filter *)filter;
strbuf_release(&ident->left);
free(filter);
}
static struct stream_filter_vtbl ident_vtbl = {
ident_filter_fn,
ident_free_fn,
};
static struct stream_filter *ident_filter(const unsigned char *sha1)
{
struct ident_filter *ident = xmalloc(sizeof(*ident));
sprintf(ident->ident, ": %s $", sha1_to_hex(sha1));
strbuf_init(&ident->left, 0);
ident->filter.vtbl = &ident_vtbl;
ident->state = 0;
return (struct stream_filter *)ident;
}
/*
* Return an appropriately constructed filter for the path, or NULL if
* the contents cannot be filtered without reading the whole thing
* in-core.
*
* Note that you would be crazy to set CRLF, smuge/clean or ident to a
* large binary blob you would want us not to slurp into the memory!
*/
struct stream_filter *get_stream_filter(const char *path, const unsigned char *sha1)
{
struct conv_attrs ca;
enum crlf_action crlf_action;
struct stream_filter *filter = NULL;
convert_attrs(&ca, path);
if (ca.ident ||
(ca.drv && (ca.drv->smudge || ca.drv->clean)))
return 0;
if (ca.drv && (ca.drv->smudge || ca.drv->clean))
return filter;
if (ca.ident)
filter = ident_filter(sha1);
crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
if ((crlf_action == CRLF_BINARY) || (crlf_action == CRLF_INPUT) ||
(crlf_action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE))
return 1;
return 0;
filter = cascade_filter(filter, &null_filter_singleton);
else if (output_eol(crlf_action) == EOL_CRLF &&
!(crlf_action == CRLF_AUTO || crlf_action == CRLF_GUESS))
filter = cascade_filter(filter, &lf_to_crlf_filter_singleton);
return filter;
}
void free_stream_filter(struct stream_filter *filter)
{
filter->vtbl->free(filter);
}
int stream_filter(struct stream_filter *filter,
const char *input, size_t *isize_p,
char *output, size_t *osize_p)
{
return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
}

72
convert.h Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright (c) 2011, Google Inc.
*/
#ifndef CONVERT_H
#define CONVERT_H
enum safe_crlf {
SAFE_CRLF_FALSE = 0,
SAFE_CRLF_FAIL = 1,
SAFE_CRLF_WARN = 2
};
extern enum safe_crlf safe_crlf;
enum auto_crlf {
AUTO_CRLF_FALSE = 0,
AUTO_CRLF_TRUE = 1,
AUTO_CRLF_INPUT = -1
};
extern enum auto_crlf auto_crlf;
enum eol {
EOL_UNSET,
EOL_CRLF,
EOL_LF,
#ifdef NATIVE_CRLF
EOL_NATIVE = EOL_CRLF
#else
EOL_NATIVE = EOL_LF
#endif
};
extern enum eol core_eol;
/* returns 1 if *dst was used */
extern int convert_to_git(const char *path, const char *src, size_t len,
struct strbuf *dst, enum safe_crlf checksafe);
extern int convert_to_working_tree(const char *path, const char *src,
size_t len, struct strbuf *dst);
extern int renormalize_buffer(const char *path, const char *src, size_t len,
struct strbuf *dst);
/*****************************************************************
*
* Streaming converison support
*
*****************************************************************/
struct stream_filter; /* opaque */
extern struct stream_filter *get_stream_filter(const char *path, const unsigned char *);
extern void free_stream_filter(struct stream_filter *);
extern int is_null_stream_filter(struct stream_filter *);
/*
* Use as much input up to *isize_p and fill output up to *osize_p;
* update isize_p and osize_p to indicate how much buffer space was
* consumed and filled. Return 0 on success, non-zero on error.
*
* Some filters may need to buffer the input and look-ahead inside it
* to decide what to output, and they may consume more than zero bytes
* of input and still not produce any output. After feeding all the
* input, pass NULL as input and keep calling this function, to let
* such filters know there is no more input coming and it is time for
* them to produce the remaining output based on the buffered input.
*/
extern int stream_filter(struct stream_filter *,
const char *input, size_t *isize_p,
char *output, size_t *osize_p);
#endif /* CONVERT_H */

12
entry.c
View File

@ -116,6 +116,7 @@ static int fstat_output(int fd, const struct checkout *state, struct stat *st)
}
static int streaming_write_entry(struct cache_entry *ce, char *path,
struct stream_filter *filter,
const struct checkout *state, int to_tempfile,
int *fstat_done, struct stat *statbuf)
{
@ -126,7 +127,7 @@ static int streaming_write_entry(struct cache_entry *ce, char *path,
ssize_t kept = 0;
int fd = -1;
st = open_istream(ce->sha1, &type, &sz);
st = open_istream(ce->sha1, &type, &sz, filter);
if (!st)
return -1;
if (type != OBJ_BLOB)
@ -186,11 +187,14 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
size_t wrote, newsize = 0;
struct stat st;
if ((ce_mode_s_ifmt == S_IFREG) &&
can_bypass_conversion(path) &&
!streaming_write_entry(ce, path, state, to_tempfile,
if (ce_mode_s_ifmt == S_IFREG) {
struct stream_filter *filter = get_stream_filter(path, ce->sha1);
if (filter &&
!streaming_write_entry(ce, path, filter,
state, to_tempfile,
&fstat_done, &st))
goto finish;
}
switch (ce_mode_s_ifmt) {
case S_IFREG:

120
streaming.c
View File

@ -41,6 +41,9 @@ struct stream_vtbl {
static open_method_decl(incore);
static open_method_decl(loose);
static open_method_decl(pack_non_delta);
static struct git_istream *attach_stream_filter(struct git_istream *st,
struct stream_filter *filter);
static open_istream_fn open_istream_tbl[] = {
open_istream_incore,
@ -48,6 +51,18 @@ static open_istream_fn open_istream_tbl[] = {
open_istream_pack_non_delta,
};
#define FILTER_BUFFER (1024*16)
struct filtered_istream {
struct git_istream *upstream;
struct stream_filter *filter;
char ibuf[FILTER_BUFFER];
char obuf[FILTER_BUFFER];
int i_end, i_ptr;
int o_end, o_ptr;
int input_finished;
};
struct git_istream {
const struct stream_vtbl *vtbl;
unsigned long size; /* inflated size of full object */
@ -72,6 +87,8 @@ struct git_istream {
struct packed_git *pack;
off_t pos;
} in_pack;
struct filtered_istream filtered;
} u;
};
@ -112,7 +129,8 @@ static enum input_source istream_source(const unsigned char *sha1,
struct git_istream *open_istream(const unsigned char *sha1,
enum object_type *type,
unsigned long *size)
unsigned long *size,
struct stream_filter *filter)
{
struct git_istream *st;
struct object_info oi;
@ -129,6 +147,14 @@ struct git_istream *open_istream(const unsigned char *sha1,
return NULL;
}
}
if (st && filter) {
/* Add "&& !is_null_stream_filter(filter)" for performance */
struct git_istream *nst = attach_stream_filter(st, filter);
if (!nst)
close_istream(st);
st = nst;
}
*size = st->size;
return st;
}
@ -147,6 +173,98 @@ static void close_deflated_stream(struct git_istream *st)
}
/*****************************************************************
*
* Filtered stream
*
*****************************************************************/
static close_method_decl(filtered)
{
free_stream_filter(st->u.filtered.filter);
return close_istream(st->u.filtered.upstream);
}
static read_method_decl(filtered)
{
struct filtered_istream *fs = &(st->u.filtered);
size_t filled = 0;
while (sz) {
/* do we already have filtered output? */
if (fs->o_ptr < fs->o_end) {
size_t to_move = fs->o_end - fs->o_ptr;
if (sz < to_move)
to_move = sz;
memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move);
fs->o_ptr += to_move;
sz -= to_move;
filled += to_move;
continue;
}
fs->o_end = fs->o_ptr = 0;
/* do we have anything to feed the filter with? */
if (fs->i_ptr < fs->i_end) {
size_t to_feed = fs->i_end - fs->i_ptr;
size_t to_receive = FILTER_BUFFER;
if (stream_filter(fs->filter,
fs->ibuf + fs->i_ptr, &to_feed,
fs->obuf, &to_receive))
return -1;
fs->i_ptr = fs->i_end - to_feed;
fs->o_end = FILTER_BUFFER - to_receive;
continue;
}
/* tell the filter to drain upon no more input */
if (fs->input_finished) {
size_t to_receive = FILTER_BUFFER;
if (stream_filter(fs->filter,
NULL, NULL,
fs->obuf, &to_receive))
return -1;
fs->o_end = FILTER_BUFFER - to_receive;
if (!fs->o_end)
break;
continue;
}
fs->i_end = fs->i_ptr = 0;
/* refill the input from the upstream */
if (!fs->input_finished) {
fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER);
if (fs->i_end < 0)
break;
if (fs->i_end)
continue;
}
fs->input_finished = 1;
}
return filled;
}
static struct stream_vtbl filtered_vtbl = {
close_istream_filtered,
read_istream_filtered,
};
static struct git_istream *attach_stream_filter(struct git_istream *st,
struct stream_filter *filter)
{
struct git_istream *ifs = xmalloc(sizeof(*ifs));
struct filtered_istream *fs = &(ifs->u.filtered);
ifs->vtbl = &filtered_vtbl;
fs->upstream = st;
fs->filter = filter;
fs->i_end = fs->i_ptr = 0;
fs->o_end = fs->o_ptr = 0;
fs->input_finished = 0;
ifs->size = -1; /* unknown */
return ifs;
}
/*****************************************************************
*
* Loose object stream

2
streaming.h
View File

@ -8,7 +8,7 @@
/* opaque */
struct git_istream;
extern struct git_istream *open_istream(const unsigned char *, enum object_type *, unsigned long *);
extern struct git_istream *open_istream(const unsigned char *, enum object_type *, unsigned long *, struct stream_filter *);
extern int close_istream(struct git_istream *);
extern ssize_t read_istream(struct git_istream *, char *, size_t);

47
t/t0021-conversion.sh
View File

@ -66,31 +66,48 @@ test_expect_success expanded_in_repo '
echo "\$Id:NoSpaceAtEitherEnd\$"
echo "\$Id: NoTerminatingSymbol"
echo "\$Id: Foreign Commit With Spaces \$"
echo "\$Id: NoTerminatingSymbolAtEOF"
} >expanded-keywords.0 &&
{
cat expanded-keywords.0 &&
printf "\$Id: NoTerminatingSymbolAtEOF"
} >expanded-keywords &&
cat expanded-keywords >expanded-keywords-crlf &&
git add expanded-keywords expanded-keywords-crlf &&
git commit -m "File with keywords expanded" &&
id=$(git rev-parse --verify :expanded-keywords) &&
{
echo "File with expanded keywords"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: fd0478f5f1486f3d5177d4c3f6eb2765e8fc56b9 \$"
echo "\$Id: $id \$"
echo "\$Id: $id \$"
echo "\$Id: $id \$"
echo "\$Id: $id \$"
echo "\$Id: $id \$"
echo "\$Id: $id \$"
echo "\$Id: NoTerminatingSymbol"
echo "\$Id: Foreign Commit With Spaces \$"
echo "\$Id: NoTerminatingSymbolAtEOF"
} >expected-output.0 &&
{
cat expected-output.0 &&
printf "\$Id: NoTerminatingSymbolAtEOF"
} >expected-output &&
{
append_cr <expected-output.0 &&
printf "\$Id: NoTerminatingSymbolAtEOF"
} >expected-output-crlf &&
{
echo "expanded-keywords ident"
echo "expanded-keywords-crlf ident text eol=crlf"
} >>.gitattributes &&
git add expanded-keywords &&
git commit -m "File with keywords expanded" &&
rm -f expanded-keywords expanded-keywords-crlf &&
echo "expanded-keywords ident" >> .gitattributes &&
rm -f expanded-keywords &&
git checkout -- expanded-keywords &&
cat expanded-keywords &&
cmp expanded-keywords expected-output
test_cmp expanded-keywords expected-output &&
git checkout -- expanded-keywords-crlf &&
test_cmp expanded-keywords-crlf expected-output-crlf
'
# The use of %f in a filter definition is expanded to the path to