2021-07-26 05:31:22 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2021-07-28 05:45:52 +00:00
|
|
|
"fmt"
|
2021-07-26 05:31:22 +00:00
|
|
|
"strings"
|
|
|
|
"unicode"
|
|
|
|
)
|
|
|
|
|
|
|
|
func eatSingleLineComment(r *strings.Reader) {
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating whitespace")
|
|
|
|
}
|
|
|
|
if c == '\n' {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatWhiteSpace(r *strings.Reader) {
|
2021-07-27 05:47:24 +00:00
|
|
|
for ;r.Len() > 0; {
|
2021-07-26 05:31:22 +00:00
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating whitespace")
|
|
|
|
}
|
|
|
|
if !unicode.IsSpace(c) {
|
2021-07-27 05:47:24 +00:00
|
|
|
r.UnreadRune()
|
2021-07-26 05:31:22 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatString(r *strings.Reader) string {
|
|
|
|
var buf strings.Builder
|
|
|
|
buf.WriteRune('"')
|
|
|
|
previous := '"'
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating string")
|
|
|
|
}
|
|
|
|
buf.WriteRune(c)
|
|
|
|
if c == '"' && previous != '\\' {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
previous = c
|
|
|
|
}
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatSymbol(r *strings.Reader) string {
|
|
|
|
var buf strings.Builder
|
|
|
|
var previous rune
|
|
|
|
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
panic("Lexing error while eating symbol")
|
|
|
|
}
|
|
|
|
if c == ')' {
|
|
|
|
r.UnreadRune()
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
if unicode.IsSpace(c) && previous != '\\' {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
buf.WriteRune(c)
|
|
|
|
previous = c
|
|
|
|
}
|
|
|
|
return buf.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func eatQuasiQuote(r *strings.Reader) []string {
|
|
|
|
tokens := make([]string, 0, 10)
|
|
|
|
tokens = append(tokens, "(", "quote")
|
|
|
|
list := false
|
|
|
|
depth := 0
|
|
|
|
var currentString string
|
|
|
|
|
|
|
|
Loop:
|
|
|
|
for r.Len() > 0 {
|
|
|
|
c, _, err := r.ReadRune()
|
|
|
|
if err != nil {
|
2021-07-27 05:47:24 +00:00
|
|
|
break Loop
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if unicode.IsSpace(c) && !list {
|
|
|
|
panic("Tokenization error: invalid construct '` '")
|
|
|
|
}
|
|
|
|
|
|
|
|
switch c {
|
|
|
|
case '(':
|
|
|
|
list = true
|
|
|
|
depth++
|
|
|
|
tokens = append(tokens, string(c))
|
|
|
|
case ')':
|
|
|
|
depth--
|
|
|
|
tokens = append(tokens, string(c))
|
|
|
|
if depth == 0 {
|
|
|
|
break Loop
|
|
|
|
}
|
|
|
|
case '`':
|
|
|
|
t := eatQuasiQuote(r)
|
|
|
|
tokens = append(tokens, t...)
|
|
|
|
case '"':
|
|
|
|
currentString = eatString(r)
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
if !list {
|
|
|
|
break Loop
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
r.UnreadRune()
|
|
|
|
currentString = eatSymbol(r)
|
|
|
|
if currentString != "" {
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
}
|
|
|
|
if !list {
|
|
|
|
break Loop
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
tokens = append(tokens, ")")
|
|
|
|
return tokens
|
|
|
|
}
|
|
|
|
|
|
|
|
func Tokenize(s string) []string {
|
|
|
|
reader := strings.NewReader(s)
|
|
|
|
tokens := make([]string, 0)
|
|
|
|
var currentString string
|
2021-07-27 05:47:24 +00:00
|
|
|
TokenizationLoop:
|
|
|
|
for ;reader.Len() > 0; {
|
2021-07-26 05:31:22 +00:00
|
|
|
c, _, err := reader.ReadRune()
|
|
|
|
if err != nil {
|
2021-07-27 05:47:24 +00:00
|
|
|
break TokenizationLoop
|
2021-07-26 05:31:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if unicode.IsSpace(c) {
|
|
|
|
eatWhiteSpace(reader)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
switch c {
|
|
|
|
case '(', ')':
|
|
|
|
tokens = append(tokens, string(c))
|
|
|
|
case '`':
|
|
|
|
t := eatQuasiQuote(reader)
|
|
|
|
tokens = append(tokens, t...)
|
|
|
|
case '"':
|
|
|
|
currentString = eatString(reader)
|
|
|
|
tokens = append(tokens, currentString)
|
2021-07-28 05:45:52 +00:00
|
|
|
case ';':
|
2021-07-26 05:31:22 +00:00
|
|
|
eatSingleLineComment(reader)
|
2021-07-28 05:45:52 +00:00
|
|
|
case '#':
|
|
|
|
c, _, err := reader.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
break TokenizationLoop
|
|
|
|
}
|
|
|
|
tokens = append(tokens, fmt.Sprintf("#%c", c))
|
2021-07-26 05:31:22 +00:00
|
|
|
default:
|
|
|
|
reader.UnreadRune()
|
|
|
|
currentString = eatSymbol(reader)
|
|
|
|
if currentString != "" {
|
|
|
|
tokens = append(tokens, currentString)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-07-27 05:47:24 +00:00
|
|
|
|
|
|
|
|
2021-07-26 05:31:22 +00:00
|
|
|
return tokens
|
|
|
|
}
|