summaryrefslogtreecommitdiff
path: root/module/language/ecmascript/tokenize.scm
diff options
context:
space:
mode:
authorAndy Wingo <wingo@pobox.com>2009-02-22 11:37:55 +0100
committerAndy Wingo <wingo@pobox.com>2009-02-22 11:37:55 +0100
commita3e34104db5eded9e48f527ef54fe6c8d18a07f8 (patch)
treee5bf4f98e8f2929439fe260e25a5c19a22d7a502 /module/language/ecmascript/tokenize.scm
parent0b229e81a7420c55bd42c1b30d86d52bf9f32cb9 (diff)
downloadguile-a3e34104db5eded9e48f527ef54fe6c8d18a07f8.tar.gz
ecmascript tokens have source info
* module/language/ecmascript/tokenize.scm: Attach source information to tokens. We have to enhance the lalr parser to actually let this information propagate through, though...
Diffstat (limited to 'module/language/ecmascript/tokenize.scm')
-rw-r--r--module/language/ecmascript/tokenize.scm69
1 files changed, 38 insertions, 31 deletions
diff --git a/module/language/ecmascript/tokenize.scm b/module/language/ecmascript/tokenize.scm
index e90a8ab12..adebd8ebd 100644
--- a/module/language/ecmascript/tokenize.scm
+++ b/module/language/ecmascript/tokenize.scm
@@ -381,37 +381,44 @@
(syntax-error "bad syntax: character not allowed" c)))))))
(define (next-token port div?)
- (let ((c (peek-char port)))
- (case c
- ((#\ht #\vt #\np #\space)
- ; whitespace
- (read-char port)
- (next-token port div?))
- ((#\newline #\cr)
- ; line break
- (read-char port)
- (next-token port div?))
- ((#\/)
- ;; division, single comment, double comment, or regexp
- (read-slash port div?))
- ((#\" #\')
- ; string literal
- `(StringLiteral . ,(read-string port)))
- (else
- (cond
- ((eof-object? c)
- '*eoi*)
- ((or (char-alphabetic? c)
- (char=? c #\$)
- (char=? c #\_))
- ;; reserved word or identifier
- (read-identifier port))
- ((char-numeric? c)
- ;; numeric -- also accept . FIXME, requires lookahead
- `(NumericLiteral . ,(read-numeric port)))
- (else
- ;; punctuation
- (read-punctuation port)))))))
+ (let ((c (peek-char port))
+ (props `((filename . ,(port-filename port))
+ (line . ,(port-line port))
+ (column . ,(port-column port)))))
+ (let ((tok
+ (case c
+ ((#\ht #\vt #\np #\space)
+ ; whitespace
+ (read-char port)
+ (next-token port div?))
+ ((#\newline #\cr)
+ ; line break
+ (read-char port)
+ (next-token port div?))
+ ((#\/)
+ ;; division, single comment, double comment, or regexp
+ (read-slash port div?))
+ ((#\" #\')
+ ; string literal
+ `(StringLiteral . ,(read-string port)))
+ (else
+ (cond
+ ((eof-object? c)
+ '*eoi*)
+ ((or (char-alphabetic? c)
+ (char=? c #\$)
+ (char=? c #\_))
+ ;; reserved word or identifier
+ (read-identifier port))
+ ((char-numeric? c)
+ ;; numeric -- also accept . FIXME, requires lookahead
+ `(NumericLiteral . ,(read-numeric port)))
+ (else
+ ;; punctuation
+ (read-punctuation port)))))))
+ (if (pair? tok)
+ (set-source-properties! tok props))
+ tok)))
(define (make-tokenizer port)
(let ((div? #f))