diff options
author | Andy Wingo <wingo@pobox.com> | 2013-01-30 15:30:31 +0100 |
---|---|---|
committer | Andy Wingo <wingo@pobox.com> | 2013-01-30 15:30:31 +0100 |
commit | 419c8736243e78a60020f5215ab223af6e9b7bb1 (patch) | |
tree | 76f0345d3e259239b35ad0141a2dcec21880f3cd | |
parent | e10c250928bc6c4116d6344616d39f3c52edc36b (diff) | |
download | guile-419c8736243e78a60020f5215ab223af6e9b7bb1.tar.gz |
Revert "detect and consume byte-order marks for textual ports"
This reverts commit b2cb557d75e4daf8c7c8cd43313f4cc51d9a3f1b, which was
pushed accidentally.
-rw-r--r-- | libguile/fports.c | 35 | ||||
-rw-r--r-- | libguile/load.c | 3 | ||||
-rw-r--r-- | libguile/ports.c | 85 | ||||
-rw-r--r-- | libguile/ports.h | 3 | ||||
-rw-r--r-- | libguile/read.c | 14 | ||||
-rw-r--r-- | test-suite/tests/filesys.test | 59 |
6 files changed, 30 insertions, 169 deletions
diff --git a/libguile/fports.c b/libguile/fports.c index fbc053031..10cf6713a 100644 --- a/libguile/fports.c +++ b/libguile/fports.c @@ -1,5 +1,5 @@ /* Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, - * 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc. + * 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -399,7 +399,7 @@ SCM_DEFINE (scm_open_file, "open-file", 2, 0, 0, #define FUNC_NAME s_scm_open_file { SCM port; - int fdes, flags = 0, scan_for_encoding = 0, consume_bom = 0, binary = 0; + int fdes, flags = 0, use_encoding = 1; unsigned int retries; char *file, *md, *ptr; @@ -415,8 +415,6 @@ SCM_DEFINE (scm_open_file, "open-file", 2, 0, 0, { case 'r': flags |= O_RDONLY; - consume_bom = 1; - scan_for_encoding = 1; break; case 'w': flags |= O_WRONLY | O_CREAT | O_TRUNC; @@ -434,12 +432,9 @@ SCM_DEFINE (scm_open_file, "open-file", 2, 0, 0, { case '+': flags = (flags & ~(O_RDONLY | O_WRONLY)) | O_RDWR; - consume_bom = 0; break; case 'b': - scan_for_encoding = 0; - consume_bom = 0; - binary = 1; + use_encoding = 0; #if defined (O_BINARY) flags |= O_BINARY; #endif @@ -478,21 +473,21 @@ SCM_DEFINE (scm_open_file, "open-file", 2, 0, 0, port = scm_i_fdes_to_port (fdes, scm_i_mode_bits (mode), fport_canonicalize_filename (filename)); - if (consume_bom) - scm_consume_byte_order_mark (port); - - if (binary) + if (use_encoding) + { + /* If this file has a coding declaration, use that as the port + encoding. */ + if (SCM_INPUT_PORT_P (port)) + { + char *enc = scm_i_scan_for_encoding (port); + if (enc != NULL) + scm_i_set_port_encoding_x (port, enc); + } + } + else /* If this is a binary file, use the binary-friendly ISO-8859-1 encoding. */ scm_i_set_port_encoding_x (port, NULL); - else if (scan_for_encoding) - /* If this is an input port and the file has a coding declaration, - use that as the port encoding. */ - { - char *enc = scm_i_scan_for_encoding (port); - if (enc != NULL) - scm_i_set_port_encoding_x (port, enc); - } scm_dynwind_end (); diff --git a/libguile/load.c b/libguile/load.c index 476461cc9..84b670549 100644 --- a/libguile/load.c +++ b/libguile/load.c @@ -106,9 +106,6 @@ SCM_DEFINE (scm_primitive_load, "primitive-load", 1, 0, 0, scm_dynwind_begin (SCM_F_DYNWIND_REWINDABLE); scm_i_dynwind_current_load_port (port); - /* FIXME: For better or for worse, scm_open_file already scans the - file for an encoding. This scans again; necessary for this - logic, but unnecessary overall. */ encoding = scm_i_scan_for_encoding (port); if (encoding) scm_i_set_port_encoding_x (port, encoding); diff --git a/libguile/ports.c b/libguile/ports.c index 9b1be9bf2..55808e272 100644 --- a/libguile/ports.c +++ b/libguile/ports.c @@ -1,5 +1,5 @@ /* Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, - * 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc. + * 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -2153,89 +2153,6 @@ SCM_DEFINE (scm_set_port_filename_x, "set-port-filename!", 2, 0, 0, } #undef FUNC_NAME -SCM_DEFINE (scm_consume_byte_order_mark, "consume-byte-order-mark", 1, 0, 0, - (SCM port), - "Peek ahead in @var{port} for a byte-order mark (\\uFEFF) encoded\n" - "in UTF-8 or in UTF-16. If found, consume the byte-order mark\n" - "and set the port to the indicated encoding.\n" - "\n" - "As a special case, if the port's encoding is already UTF-16LE\n" - "or UTF-16BE (as opposed to UTF-16), we consider that the user\n" - "has already asked for an explicit byte order. In this case no\n" - "scan is performed, and the byte-order mark (if any) is left in\n" - "the port.\n" - "\n" - "Return @code{#t} if a byte-order mark was consumed, and\n" - "@code{#f} otherwise.") -#define FUNC_NAME s_scm_consume_byte_order_mark -{ - scm_t_port *pt; - const char *enc; - - SCM_VALIDATE_PORT (1, port); - - pt = SCM_PTAB_ENTRY (port); - enc = pt->encoding; - - if (enc && strcasecmp (enc, "UTF-16BE") == 0) - return SCM_BOOL_F; - - if (enc && strcasecmp (enc, "UTF-16LE") == 0) - return SCM_BOOL_F; - - switch (scm_peek_byte_or_eof (port)) - { - case 0xEF: - scm_get_byte_or_eof (port); - switch (scm_peek_byte_or_eof (port)) - { - case 0xBB: - scm_get_byte_or_eof (port); - switch (scm_peek_byte_or_eof (port)) - { - case 0xBF: - scm_get_byte_or_eof (port); - scm_i_set_port_encoding_x (port, "UTF-8"); - return SCM_BOOL_T; - default: - scm_unget_byte (0xBB, port); - scm_unget_byte (0xEF, port); - return SCM_BOOL_F; - } - default: - scm_unget_byte (0xEF, port); - return SCM_BOOL_F; - } - case 0xFE: - scm_get_byte_or_eof (port); - switch (scm_peek_byte_or_eof (port)) - { - case 0xFF: - scm_get_byte_or_eof (port); - scm_i_set_port_encoding_x (port, "UTF-16BE"); - return SCM_BOOL_T; - default: - scm_unget_byte (0xFE, port); - return SCM_BOOL_F; - } - case 0xFF: - scm_get_byte_or_eof (port); - switch (scm_peek_byte_or_eof (port)) - { - case 0xFE: - scm_get_byte_or_eof (port); - scm_i_set_port_encoding_x (port, "UTF-16LE"); - return SCM_BOOL_T; - default: - scm_unget_byte (0xFF, port); - return SCM_BOOL_F; - } - default: - return SCM_BOOL_F; - } -} -#undef FUNC_NAME - /* A fluid specifying the default encoding for newly created ports. If it is a string, that is the encoding. If it is #f, it is in the "native" (Latin-1) encoding. */ diff --git a/libguile/ports.h b/libguile/ports.h index 2f3234530..d4d59b70e 100644 --- a/libguile/ports.h +++ b/libguile/ports.h @@ -4,7 +4,7 @@ #define SCM_PORTS_H /* Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, - * 2006, 2008, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc. + * 2006, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -303,7 +303,6 @@ SCM_API SCM scm_port_column (SCM port); SCM_API SCM scm_set_port_column_x (SCM port, SCM line); SCM_API SCM scm_port_filename (SCM port); SCM_API SCM scm_set_port_filename_x (SCM port, SCM filename); -SCM_API SCM scm_consume_byte_order_mark (SCM port); SCM_INTERNAL const char *scm_i_default_port_encoding (void); SCM_INTERNAL void scm_i_set_default_port_encoding (const char *); SCM_INTERNAL void scm_i_set_port_encoding_x (SCM port, const char *str); diff --git a/libguile/read.c b/libguile/read.c index a8f77449f..222891b7f 100644 --- a/libguile/read.c +++ b/libguile/read.c @@ -1,5 +1,5 @@ /* Copyright (C) 1995, 1996, 1997, 1999, 2000, 2001, 2003, 2004, 2006, - * 2007, 2008, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc. + * 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -1985,6 +1985,7 @@ scm_i_scan_for_encoding (SCM port) char header[SCM_ENCODING_SEARCH_SIZE+1]; size_t bytes_read, encoding_length, i; char *encoding = NULL; + int utf8_bom = 0; char *pos, *encoding_start; int in_comment; @@ -2026,9 +2027,13 @@ scm_i_scan_for_encoding (SCM port) bytes_read = scm_c_read (port, header, SCM_ENCODING_SEARCH_SIZE); header[bytes_read] = '\0'; - scm_seek (port, scm_from_int (-bytes_read), scm_from_int (SEEK_CUR)); + scm_seek (port, scm_from_int (0), scm_from_int (SEEK_SET)); } + if (bytes_read > 3 + && header[0] == '\xef' && header[1] == '\xbb' && header[2] == '\xbf') + utf8_bom = 1; + /* search past "coding[:=]" */ pos = header; while (1) @@ -2097,6 +2102,11 @@ scm_i_scan_for_encoding (SCM port) /* This wasn't in a comment */ return NULL; + if (utf8_bom && strcmp(encoding, "UTF-8")) + scm_misc_error (NULL, + "the port input declares the encoding ~s but is encoded as UTF-8", + scm_list_1 (scm_from_locale_string (encoding))); + return encoding; } diff --git a/test-suite/tests/filesys.test b/test-suite/tests/filesys.test index 8bd974dbd..a6bfb6eb5 100644 --- a/test-suite/tests/filesys.test +++ b/test-suite/tests/filesys.test @@ -1,6 +1,6 @@ ;;;; filesys.test --- test file system functions -*- scheme -*- ;;;; -;;;; Copyright (C) 2004, 2006, 2013 Free Software Foundation, Inc. +;;;; Copyright (C) 2004, 2006 Free Software Foundation, Inc. ;;;; ;;;; This library is free software; you can redistribute it and/or ;;;; modify it under the terms of the GNU Lesser General Public @@ -17,8 +17,6 @@ ;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA (define-module (test-suite test-filesys) - #:use-module (ice-9 rdelim) - #:use-module (ice-9 binary-ports) #:use-module (test-suite lib) #:use-module (test-suite guile-test)) @@ -129,58 +127,3 @@ (delete-file (test-file)) (delete-file (test-symlink)) - -(let ((s "\ufeffHello, world!")) - (define* (test-encoding encoding #:optional (ambient "ISO-8859-1")) - (with-fluids ((%default-port-encoding ambient)) - (let* ((bytes (catch 'misc-error - (lambda () - (call-with-values open-bytevector-output-port - (lambda (port get-bytevector) - (set-port-encoding! port encoding) - (display s port) - (get-bytevector)))) - (lambda args - (throw 'unresolved)))) - (name (string-copy "myfile-XXXXXX")) - (port (mkstemp! name))) - (put-bytevector port bytes) - (close-port port) - (let ((contents (call-with-input-file name read-string))) - (delete-file name) - contents)))) - - (pass-if "UTF-8" - (equal? (test-encoding "UTF-8") - "Hello, world!")) - - (pass-if "UTF-16BE" - (equal? (test-encoding "UTF-16BE") - "Hello, world!")) - - (pass-if "UTF-16LE" - (equal? (test-encoding "UTF-16LE") - "Hello, world!")) - - (pass-if "UTF-8 (ambient)" - (equal? (test-encoding "UTF-8" "UTF-8") - "Hello, world!")) - - (pass-if "UTF-8 (UTF-16 ambient)" - (equal? (test-encoding "UTF-8" "UTF-16") - "Hello, world!")) - - ;; Unicode 6.2 section 16.8: - ;; - ;; For compatibility with versions of the Unicode Standard prior to - ;; Version 3.2, the code point U+FEFF has the word-joining semantics - ;; of zero width no-break space when it is not used as a BOM. [...] - ;; - ;; Where the byte order is explicitly specified, such as in UTF-16BE - ;; or UTF-16LE, then all U+FEFF characters -- even at the very - ;; beginning of the text -- are to be interpreted as zero width - ;; no-break spaces. - ;; - (pass-if "UTF-16LE (ambient)" - (equal? (test-encoding "UTF-16LE" "UTF-16LE") - "\ufeffHello, world!"))) |