diff options
author | Ludovic Courtès <ludo@gnu.org> | 2011-05-07 22:46:38 +0200 |
---|---|---|
committer | Ludovic Courtès <ludo@gnu.org> | 2011-05-07 22:47:49 +0200 |
commit | 7be1705dbda377780335ecbcbfce04de523f2671 (patch) | |
tree | d198136d5e6fc21759c5e9c9a4b008b4e421971a /libguile/ports.c | |
parent | 452c5ad912baee9fa64298b6a8905681557ad3ae (diff) | |
download | guile-7be1705dbda377780335ecbcbfce04de523f2671.tar.gz |
Fix `get_utf8_codepoint' to not consume valid starting bytes.
Thanks to Mark H. Weaver for pointing this out.
* libguile/ports.c (CONSUME_PEEKED_BYTE): New macro.
(get_utf8_codepoint): New variable `pt'. Use
`scm_peek_byte_or_eof'/`CONSUME_PEEKED_BYTE' pairs instead of
`scm_get_byte_or_eof'.
* test-suite/tests/ports.test ("string ports")[#xc2 #x41 #x42, #xe0 #xa0
#x41 #x42, #xf0 #x88 #x88 #x88]: Fix to conform to Unicode 6.0.0.
[#xe0 #x88 #x88]: Remove test.
[#xf0 #x80 #x80 #x41]: New test.
Diffstat (limited to 'libguile/ports.c')
-rw-r--r-- | libguile/ports.c | 92 |
1 files changed, 42 insertions, 50 deletions
diff --git a/libguile/ports.c b/libguile/ports.c index 767e08640..926149bf9 100644 --- a/libguile/ports.c +++ b/libguile/ports.c @@ -1127,10 +1127,14 @@ get_utf8_codepoint (SCM port, scm_t_wchar *codepoint, #define ASSERT_NOT_EOF(b) \ if (SCM_UNLIKELY ((b) == EOF)) \ goto invalid_seq +#define CONSUME_PEEKED_BYTE() \ + pt->read_pos++ int byte; + scm_t_port *pt; *len = 0; + pt = SCM_PTAB_ENTRY (port); byte = scm_get_byte_or_eof (port); if (byte == EOF) @@ -1148,49 +1152,44 @@ get_utf8_codepoint (SCM port, scm_t_wchar *codepoint, else if (buf[0] >= 0xc2 && buf[0] <= 0xdf) { /* 2-byte form. */ - byte = scm_get_byte_or_eof (port); + byte = scm_peek_byte_or_eof (port); ASSERT_NOT_EOF (byte); - buf[1] = (scm_t_uint8) byte; - *len = 2; - if (SCM_UNLIKELY ((byte & 0xc0) != 0x80)) goto invalid_seq; + CONSUME_PEEKED_BYTE (); + buf[1] = (scm_t_uint8) byte; + *len = 2; + *codepoint = ((scm_t_wchar) buf[0] & 0x1f) << 6UL | (buf[1] & 0x3f); } else if ((buf[0] & 0xf0) == 0xe0) { /* 3-byte form. */ - byte = scm_get_byte_or_eof (port); - if (SCM_UNLIKELY (byte == EOF)) - goto invalid_seq; - - buf[1] = (scm_t_uint8) byte; - *len = 2; + byte = scm_peek_byte_or_eof (port); + ASSERT_NOT_EOF (byte); if (SCM_UNLIKELY ((byte & 0xc0) != 0x80 || (buf[0] == 0xe0 && byte < 0xa0) || (buf[0] == 0xed && byte > 0x9f))) - { - /* Swallow the 3rd byte. */ - byte = scm_get_byte_or_eof (port); - ASSERT_NOT_EOF (byte); - *len = 3, buf[2] = byte; - goto invalid_seq; - } + goto invalid_seq; + CONSUME_PEEKED_BYTE (); + buf[1] = (scm_t_uint8) byte; + *len = 2; - byte = scm_get_byte_or_eof (port); + byte = scm_peek_byte_or_eof (port); ASSERT_NOT_EOF (byte); - buf[2] = (scm_t_uint8) byte; - *len = 3; - if (SCM_UNLIKELY ((byte & 0xc0) != 0x80)) goto invalid_seq; + CONSUME_PEEKED_BYTE (); + buf[2] = (scm_t_uint8) byte; + *len = 3; + *codepoint = ((scm_t_wchar) buf[0] & 0x0f) << 12UL | ((scm_t_wchar) buf[1] & 0x3f) << 6UL | (buf[2] & 0x3f); @@ -1198,51 +1197,38 @@ get_utf8_codepoint (SCM port, scm_t_wchar *codepoint, else if (buf[0] >= 0xf0 && buf[0] <= 0xf4) { /* 4-byte form. */ - byte = scm_get_byte_or_eof (port); + byte = scm_peek_byte_or_eof (port); ASSERT_NOT_EOF (byte); - buf[1] = (scm_t_uint8) byte; - *len = 2; - if (SCM_UNLIKELY (((byte & 0xc0) != 0x80) || (buf[0] == 0xf0 && byte < 0x90) || (buf[0] == 0xf4 && byte > 0x8f))) - { - /* Swallow the 3rd and 4th bytes. */ - byte = scm_get_byte_or_eof (port); - ASSERT_NOT_EOF (byte); - *len = 3, buf[2] = byte; - - byte = scm_get_byte_or_eof (port); - ASSERT_NOT_EOF (byte); - *len = 4, buf[3] = byte; - goto invalid_seq; - } + goto invalid_seq; - byte = scm_get_byte_or_eof (port); + CONSUME_PEEKED_BYTE (); + buf[1] = (scm_t_uint8) byte; + *len = 2; + + byte = scm_peek_byte_or_eof (port); ASSERT_NOT_EOF (byte); + if (SCM_UNLIKELY ((byte & 0xc0) != 0x80)) + goto invalid_seq; + + CONSUME_PEEKED_BYTE (); buf[2] = (scm_t_uint8) byte; *len = 3; - if (SCM_UNLIKELY ((byte & 0xc0) != 0x80)) - { - /* Swallow the 4th byte. */ - byte = scm_get_byte_or_eof (port); - ASSERT_NOT_EOF (byte); - *len = 4, buf[3] = byte; - goto invalid_seq; - } - - byte = scm_get_byte_or_eof (port); + byte = scm_peek_byte_or_eof (port); ASSERT_NOT_EOF (byte); - buf[3] = (scm_t_uint8) byte; - *len = 4; - if (SCM_UNLIKELY ((byte & 0xc0) != 0x80)) goto invalid_seq; + CONSUME_PEEKED_BYTE (); + buf[3] = (scm_t_uint8) byte; + *len = 4; + *codepoint = ((scm_t_wchar) buf[0] & 0x07) << 18UL | ((scm_t_wchar) buf[1] & 0x3f) << 12UL | ((scm_t_wchar) buf[2] & 0x3f) << 6UL @@ -1254,8 +1240,14 @@ get_utf8_codepoint (SCM port, scm_t_wchar *codepoint, return 0; invalid_seq: + /* Here we could choose the consume the faulty byte when it's not a + valid starting byte, but it's not a requirement. What Section 3.9 + of Unicode 6.0.0 mandates, though, is to not consume a byte that + would otherwise be a valid starting byte. */ + return EILSEQ; +#undef CONSUME_PEEKED_BYTE #undef ASSERT_NOT_EOF } |