undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

utf8: add function to detect prohibited UTF-16/32 BOM

Browse Source 
Whenever a data stream is declared to be UTF-16BE, UTF-16LE, UTF-32BE
or UTF-32LE a BOM must not be used [1]. The function returns true if
this is the case.

This function is used in a subsequent commit.

[1] http://unicode.org/faq/utf_bom.html#bom10

Signed-off-by: Lars Schneider <larsxschneider@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Lars Schneider 2018-04-15 20:16:05 +02:00 committed by Junio C Hamano
parent 2f0c4a362c
commit 10ecb82e4f
2 changed files with 35 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
utf8.c
View File

@ -560,6 +560,32 @@ char *reencode_string_len(const char *in, int insz,
} }
#endif #endif
static int has_bom_prefix(const char *data, size_t len,
const char *bom, size_t bom_len)
{
return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len);
}
static const char utf16_be_bom[] = {0xFE, 0xFF};
static const char utf16_le_bom[] = {0xFF, 0xFE};
static const char utf32_be_bom[] = {0x00, 0x00, 0xFE, 0xFF};
static const char utf32_le_bom[] = {0xFF, 0xFE, 0x00, 0x00};
int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
{
return (
(same_utf_encoding("UTF-16BE", enc) ||
same_utf_encoding("UTF-16LE", enc)) &&
(has_bom_prefix(data, len, utf16_be_bom, sizeof(utf16_be_bom)) ||
has_bom_prefix(data, len, utf16_le_bom, sizeof(utf16_le_bom)))
) || (
(same_utf_encoding("UTF-32BE", enc) ||
same_utf_encoding("UTF-32LE", enc)) &&
(has_bom_prefix(data, len, utf32_be_bom, sizeof(utf32_be_bom)) ||
has_bom_prefix(data, len, utf32_le_bom, sizeof(utf32_le_bom)))
);
}
/* /*
* Returns first character length in bytes for multi-byte `text` according to * Returns first character length in bytes for multi-byte `text` according to
* `encoding`. * `encoding`.

9
utf8.h
View File

@ -70,4 +70,13 @@ typedef enum {
void strbuf_utf8_align(struct strbuf *buf, align_type position, unsigned int width, void strbuf_utf8_align(struct strbuf *buf, align_type position, unsigned int width,
const char *s); const char *s);
/*
* If a data stream is declared as UTF-16BE or UTF-16LE, then a UTF-16
* BOM must not be used [1]. The same applies for the UTF-32 equivalents.
* The function returns true if this rule is violated.
*
* [1] http://unicode.org/faq/utf_bom.html#bom10
*/
int has_prohibited_utf_bom(const char *enc, const char *data, size_t len);
#endif #endif