2021-07-26 05:31:22 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2021-07-28 05:45:52 +00:00
|
|
|
"fmt"
|
2021-07-26 05:31:22 +00:00
|
|
|
"strings"
|
|
|
|
"unicode"
|
|
|
|
)
|
|
|
|
|
2021-08-20 05:54:41 +00:00
|
|
|
var lexLine int = 1
|
|
|
|
var parens int = 0
|
|
|
|
|
2021-07-26 05:31:22 +00:00
|
|
|
func eatSingleLineComment(r *strings.Reader) {
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
2021-08-20 05:54:41 +00:00
|
|
|
panic(fmt.Sprintf("Lexing error while eating comment on line %d", lexLine))
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
if c == '\n' {
|
2021-08-20 05:54:41 +00:00
|
|
|
lexLine++
|
2021-07-26 05:31:22 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatWhiteSpace(r *strings.Reader) {
|
2021-07-31 20:25:20 +00:00
|
|
|
for r.Len() > 0 {
|
2021-07-26 05:31:22 +00:00
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating whitespace")
|
|
|
|
}
|
2021-08-20 05:54:41 +00:00
|
|
|
if c == '\n' {
|
|
|
|
lexLine++
|
|
|
|
}
|
2021-07-26 05:31:22 +00:00
|
|
|
if !unicode.IsSpace(c) {
|
2021-07-27 05:47:24 +00:00
|
|
|
r.UnreadRune()
|
2021-07-26 05:31:22 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatString(r *strings.Reader) string {
|
|
|
|
var buf strings.Builder
|
|
|
|
buf.WriteRune('"')
|
2022-07-14 03:44:31 +00:00
|
|
|
escapeCount := 0
|
2021-07-26 05:31:22 +00:00
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating string")
|
|
|
|
}
|
|
|
|
buf.WriteRune(c)
|
2022-07-14 03:44:31 +00:00
|
|
|
if c == '"' && (escapeCount == 0 || escapeCount%2 == 0) {
|
2021-07-26 05:31:22 +00:00
|
|
|
break
|
2021-09-06 22:40:15 +00:00
|
|
|
} else if c == '\n' || (c != '"' && r.Len() == 0) {
|
|
|
|
panic(fmt.Sprintf("Parse error: Unclosed string on line %d ", lexLine))
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
2022-07-14 03:44:31 +00:00
|
|
|
if c == '\\' {
|
|
|
|
escapeCount++
|
|
|
|
} else {
|
|
|
|
escapeCount = 0
|
|
|
|
}
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatSymbol(r *strings.Reader) string {
|
|
|
|
var buf strings.Builder
|
|
|
|
var previous rune
|
|
|
|
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating symbol")
|
|
|
|
}
|
2021-08-29 17:08:08 +00:00
|
|
|
if c == ')' || c == ']' {
|
2021-07-26 05:31:22 +00:00
|
|
|
r.UnreadRune()
|
|
|
|
return buf.String()
|
2021-08-20 05:54:41 +00:00
|
|
|
} else if c == '\n' {
|
|
|
|
lexLine++
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
if unicode.IsSpace(c) && previous != '\\' {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
buf.WriteRune(c)
|
|
|
|
previous = c
|
|
|
|
}
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
|
2021-08-29 17:08:08 +00:00
|
|
|
func eatQuote(r *strings.Reader) []string {
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens := make([]string, 0, 10)
|
|
|
|
tokens = append(tokens, "(", "quote")
|
2021-08-22 03:27:17 +00:00
|
|
|
parens++
|
2021-07-26 05:31:22 +00:00
|
|
|
list := false
|
|
|
|
depth := 0
|
|
|
|
var currentString string
|
|
|
|
|
2021-07-31 20:25:20 +00:00
|
|
|
Loop:
|
2021-07-26 05:31:22 +00:00
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
2021-07-27 05:47:24 +00:00
|
|
|
break Loop
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
2021-08-20 05:54:41 +00:00
|
|
|
if c == '\n' {
|
|
|
|
lexLine++
|
|
|
|
}
|
2021-07-26 05:31:22 +00:00
|
|
|
|
|
|
|
if unicode.IsSpace(c) && !list {
|
2021-08-20 05:54:41 +00:00
|
|
|
panic(fmt.Sprintf("Parse error: Invalid construct \"' \" on line %d", lexLine))
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
switch c {
|
|
|
|
case '(':
|
|
|
|
list = true
|
|
|
|
depth++
|
2021-08-20 05:54:41 +00:00
|
|
|
parens++
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens = append(tokens, string(c))
|
2021-08-29 17:08:08 +00:00
|
|
|
case '[':
|
2021-07-29 22:36:11 +00:00
|
|
|
list = true
|
|
|
|
depth++
|
2021-08-20 05:54:41 +00:00
|
|
|
parens++
|
2021-08-29 17:08:08 +00:00
|
|
|
tokens = append(tokens, "(", "list")
|
2021-07-26 05:31:22 +00:00
|
|
|
case ')':
|
|
|
|
depth--
|
2021-08-20 05:54:41 +00:00
|
|
|
parens--
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens = append(tokens, string(c))
|
|
|
|
if depth == 0 {
|
|
|
|
break Loop
|
|
|
|
}
|
2021-08-29 17:08:08 +00:00
|
|
|
case ']':
|
2021-07-29 22:36:11 +00:00
|
|
|
depth--
|
2021-08-20 05:54:41 +00:00
|
|
|
parens--
|
2021-07-29 22:36:11 +00:00
|
|
|
tokens = append(tokens, ")")
|
|
|
|
if depth == 0 {
|
|
|
|
break Loop
|
|
|
|
}
|
2021-08-02 06:06:52 +00:00
|
|
|
case '\'':
|
2021-08-29 17:08:08 +00:00
|
|
|
t := eatQuote(r)
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens = append(tokens, t...)
|
|
|
|
case '"':
|
|
|
|
currentString = eatString(r)
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
if !list {
|
|
|
|
break Loop
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
r.UnreadRune()
|
|
|
|
currentString = eatSymbol(r)
|
|
|
|
if currentString != "" {
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
}
|
|
|
|
if !list {
|
|
|
|
break Loop
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
tokens = append(tokens, ")")
|
2021-08-22 03:27:17 +00:00
|
|
|
parens--
|
2021-07-26 05:31:22 +00:00
|
|
|
return tokens
|
|
|
|
}
|
|
|
|
|
2021-08-20 05:54:41 +00:00
|
|
|
func eatShebang(r *strings.Reader) {
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Lexing error while eating shebang on line %d", lexLine))
|
|
|
|
}
|
|
|
|
if c == '\n' {
|
|
|
|
lexLine++
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-26 05:31:22 +00:00
|
|
|
func Tokenize(s string) []string {
|
2021-08-20 05:54:41 +00:00
|
|
|
lexLine = 1
|
|
|
|
parens = 0
|
2021-07-26 05:31:22 +00:00
|
|
|
reader := strings.NewReader(s)
|
|
|
|
tokens := make([]string, 0)
|
|
|
|
var currentString string
|
2021-08-20 05:54:41 +00:00
|
|
|
|
|
|
|
// Handle shebang
|
|
|
|
if reader.Len() > 2 {
|
|
|
|
c, _, _ := reader.ReadRune()
|
|
|
|
if c == '#' {
|
|
|
|
c, _, _ := reader.ReadRune()
|
|
|
|
if c == '!' {
|
|
|
|
eatShebang(reader)
|
2021-09-04 14:45:35 +00:00
|
|
|
} else {
|
2021-08-20 05:54:41 +00:00
|
|
|
reader.UnreadRune()
|
|
|
|
reader.UnreadRune()
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
reader.UnreadRune()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-31 20:25:20 +00:00
|
|
|
TokenizationLoop:
|
|
|
|
for reader.Len() > 0 {
|
2021-07-26 05:31:22 +00:00
|
|
|
c, _, err := reader.ReadRune()
|
|
|
|
if err != nil {
|
2021-07-27 05:47:24 +00:00
|
|
|
break TokenizationLoop
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if unicode.IsSpace(c) {
|
2021-08-20 05:54:41 +00:00
|
|
|
if c == '\n' {
|
|
|
|
lexLine++
|
|
|
|
}
|
2021-07-26 05:31:22 +00:00
|
|
|
eatWhiteSpace(reader)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
switch c {
|
2021-08-20 05:54:41 +00:00
|
|
|
case '(':
|
|
|
|
tokens = append(tokens, string(c))
|
|
|
|
parens++
|
|
|
|
case ')':
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens = append(tokens, string(c))
|
2021-08-20 05:54:41 +00:00
|
|
|
parens--
|
2021-08-29 17:08:08 +00:00
|
|
|
case ']':
|
2021-07-29 22:36:11 +00:00
|
|
|
tokens = append(tokens, ")")
|
2021-08-20 05:54:41 +00:00
|
|
|
parens--
|
2021-08-29 17:08:08 +00:00
|
|
|
case '[':
|
|
|
|
tokens = append(tokens, "(", "list")
|
2021-08-20 05:54:41 +00:00
|
|
|
parens++
|
2021-08-02 06:06:52 +00:00
|
|
|
case '\'':
|
2021-08-29 17:08:08 +00:00
|
|
|
t := eatQuote(reader)
|
2021-07-26 05:31:22 +00:00
|
|
|
tokens = append(tokens, t...)
|
|
|
|
case '"':
|
|
|
|
currentString = eatString(reader)
|
2022-10-16 04:35:03 +00:00
|
|
|
tokens = append(tokens, unescapeString(currentString))
|
2021-07-28 05:45:52 +00:00
|
|
|
case ';':
|
2021-07-26 05:31:22 +00:00
|
|
|
eatSingleLineComment(reader)
|
2021-07-28 05:45:52 +00:00
|
|
|
case '#':
|
|
|
|
c, _, err := reader.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
break TokenizationLoop
|
|
|
|
}
|
|
|
|
tokens = append(tokens, fmt.Sprintf("#%c", c))
|
2021-07-26 05:31:22 +00:00
|
|
|
default:
|
|
|
|
reader.UnreadRune()
|
|
|
|
currentString = eatSymbol(reader)
|
|
|
|
if currentString != "" {
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
}
|
|
|
|
}
|
2021-08-20 05:54:41 +00:00
|
|
|
|
|
|
|
if parens < 0 {
|
|
|
|
panic(fmt.Sprintf("Parse error: too many closing parenthesis near line %d", lexLine))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if parens > 0 {
|
|
|
|
panic("Parse error: parse ended without enough closing parenthesis")
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
2021-07-27 05:47:24 +00:00
|
|
|
|
2021-07-26 05:31:22 +00:00
|
|
|
return tokens
|
|
|
|
}
|