diff options
author | Andy Wingo <wingo@pobox.com> | 2009-02-22 11:37:55 +0100 |
---|---|---|
committer | Andy Wingo <wingo@pobox.com> | 2009-02-22 11:37:55 +0100 |
commit | a3e34104db5eded9e48f527ef54fe6c8d18a07f8 (patch) | |
tree | e5bf4f98e8f2929439fe260e25a5c19a22d7a502 /module/language/ecmascript/tokenize.scm | |
parent | 0b229e81a7420c55bd42c1b30d86d52bf9f32cb9 (diff) | |
download | guile-a3e34104db5eded9e48f527ef54fe6c8d18a07f8.tar.gz |
ecmascript tokens have source info
* module/language/ecmascript/tokenize.scm: Attach source information to
tokens. We have to enhance the lalr parser to actually let this
information propagate through, though...
Diffstat (limited to 'module/language/ecmascript/tokenize.scm')
-rw-r--r-- | module/language/ecmascript/tokenize.scm | 69 |
1 files changed, 38 insertions, 31 deletions
diff --git a/module/language/ecmascript/tokenize.scm b/module/language/ecmascript/tokenize.scm index e90a8ab12..adebd8ebd 100644 --- a/module/language/ecmascript/tokenize.scm +++ b/module/language/ecmascript/tokenize.scm @@ -381,37 +381,44 @@ (syntax-error "bad syntax: character not allowed" c))))))) (define (next-token port div?) - (let ((c (peek-char port))) - (case c - ((#\ht #\vt #\np #\space) - ; whitespace - (read-char port) - (next-token port div?)) - ((#\newline #\cr) - ; line break - (read-char port) - (next-token port div?)) - ((#\/) - ;; division, single comment, double comment, or regexp - (read-slash port div?)) - ((#\" #\') - ; string literal - `(StringLiteral . ,(read-string port))) - (else - (cond - ((eof-object? c) - '*eoi*) - ((or (char-alphabetic? c) - (char=? c #\$) - (char=? c #\_)) - ;; reserved word or identifier - (read-identifier port)) - ((char-numeric? c) - ;; numeric -- also accept . FIXME, requires lookahead - `(NumericLiteral . ,(read-numeric port))) - (else - ;; punctuation - (read-punctuation port))))))) + (let ((c (peek-char port)) + (props `((filename . ,(port-filename port)) + (line . ,(port-line port)) + (column . ,(port-column port))))) + (let ((tok + (case c + ((#\ht #\vt #\np #\space) + ; whitespace + (read-char port) + (next-token port div?)) + ((#\newline #\cr) + ; line break + (read-char port) + (next-token port div?)) + ((#\/) + ;; division, single comment, double comment, or regexp + (read-slash port div?)) + ((#\" #\') + ; string literal + `(StringLiteral . ,(read-string port))) + (else + (cond + ((eof-object? c) + '*eoi*) + ((or (char-alphabetic? c) + (char=? c #\$) + (char=? c #\_)) + ;; reserved word or identifier + (read-identifier port)) + ((char-numeric? c) + ;; numeric -- also accept . FIXME, requires lookahead + `(NumericLiteral . ,(read-numeric port))) + (else + ;; punctuation + (read-punctuation port))))))) + (if (pair? tok) + (set-source-properties! tok props)) + tok))) (define (make-tokenizer port) (let ((div? #f)) |