1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
;;;; elisp-reader.test --- Test the reader used by the Elisp compiler.
;;;;
;;;; Copyright (C) 2009 Free Software Foundation, Inc.
;;;; Daniel Kraft
;;;;
;;;; This library is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU Lesser General Public
;;;; License as published by the Free Software Foundation; either
;;;; version 3 of the License, or (at your option) any later version.
;;;;
;;;; This library is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;;;; Lesser General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU Lesser General Public
;;;; License along with this library; if not, write to the Free Software
;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
(define-module (test-elisp-reader)
:use-module (test-suite lib)
:use-module (language elisp lexer))
; ==============================================================================
; Test the lexer.
; This is of course somewhat redundant with the full parser checks, but probably
; can't hurt and is useful in developing the lexer itself.
(define (get-string-lexer str)
(call-with-input-string str get-lexer))
(define (lex-string str)
(let ((lexer (get-string-lexer str)))
(let iterate ((result '()))
(let ((token (lexer)))
(if (eq? token '*eoi*)
(reverse result)
(iterate (cons token result)))))))
(with-test-prefix "Lexer"
(let ((lexer (get-string-lexer "")))
(pass-if "end-of-input"
(and (eq? (lexer) '*eoi*)
(eq? (lexer) '*eoi*)
(eq? (lexer) '*eoi*))))
(pass-if "single character tokens"
(equal? (lex-string "()[]'`, . ")
'((paren-open . #f) (paren-close . #f)
(square-open . #f) (square-close . #f)
(quote . #f) (backquote . #f) (unquote . #f) (dot . #f))))
(pass-if "whitespace and comments"
(equal? (lex-string " (\n\t) ; this is a comment\n. ; until eof")
'((paren-open . #f) (paren-close . #f) (dot . #f))))
(pass-if "source properties"
(let ((x (car (lex-string "\n\n \n . \n"))))
(and (= (source-property x 'line) 4)
(= (source-property x 'column) 3))))
(pass-if "symbols"
(equal? (lex-string "foo FOO char-to-string 1+ \\+1
\\(*\\ 1\\ 2\\)
+-*/_~!@$%^&=:<>{}
abc(def)ghi .e5")
`((symbol . foo) (symbol . FOO) (symbol . char-to-string)
(symbol . 1+) (symbol . ,(string->symbol "+1"))
(symbol . ,(string->symbol "(* 1 2)"))
(symbol . +-*/_~!@$%^&=:<>{})
(symbol . abc) (paren-open . #f) (symbol . def)
(paren-close . #f) (symbol . ghi) (symbol . .e5))))
; Here we make use of the property that exact/inexact numbers are not equal?
; even when they have the same numeric value!
(pass-if "integers"
(equal? (lex-string "-1 1 1. +1 01234")
'((integer . -1) (integer . 1) (integer . 1) (integer . 1)
(integer . 1234))))
(pass-if "floats"
(equal? (lex-string "1500.0 15e2 15.e2 1.5e3 .15e4 -.345e-2")
'((float . 1500.0) (float . 1500.0) (float . 1500.0)
(float . 1500.0) (float . 1500.0)
(float . -0.00345))))
; Check string lexing, this also checks basic character escape sequences
; that are then (hopefully) also correct for character literals.
(pass-if "strings"
(equal? (lex-string "\"foo\\nbar
test\\
\\\"ab\\\"\\\\ ab\\ cd
\\418\\0415\\u0041\\U0000000A\\Xab\\x0000000000000004fG.\" ")
'((string . "foo\nbar
test\"ab\"\\ abcd
!8!5A\nXabOG."))))
(pass-if "ASCII control characters and meta in strings"
(equal? (lex-string "\"\\^?\\C-a\\C-A\\^z\\M-B\\M-\\^@\\M-\\C-a\"")
'((string . "\x7F\x01\x01\x1A\xC2\x80\x81"))))
; Character literals, taking into account that some escape sequences were
; already checked in the strings.
(pass-if "characters"
(equal? (lex-string "?A?\\z ? ?\\x21 ?\\^j ?\\\\?\\n?\\\n")
`((character . 65) (character . ,(char->integer #\z))
(character . 32) (character . ,(char->integer #\!))
(character . 10) (character . ,(char->integer #\\))
(character . 10) (character . 10))))
(pass-if "meta characters"
(equal? (map cdr (lex-string "?\\C-[?\\M-\\S-Z?\\^X?\\A-\\s-\\H-\\s"))
`(,(+ (expt 2 26) (char->integer #\[))
,(+ (expt 2 27) (expt 2 25) (char->integer #\Z))
,(- (char->integer #\X) (char->integer #\@))
,(+ (expt 2 22) (expt 2 23) (expt 2 24) 32)))))
|