slope/lexer.go

package main

import (
	"fmt"
	"strings"
	"unicode"
)

func eatSingleLineComment(r *strings.Reader) {
	for r.Len() > 0 {
		c, _, err := r.ReadRune()
		if err != nil {
			panic("Lexing error while eating whitespace")
		}
		if c == '\n' {
			break
		}
	}
}

func eatWhiteSpace(r *strings.Reader) {
	for ;r.Len() > 0; {
		c, _, err := r.ReadRune()
		if err != nil {
			panic("Lexing error while eating whitespace")
		}
		if !unicode.IsSpace(c) {
			r.UnreadRune()
			break
		}
	}
}

func eatString(r *strings.Reader) string {
	var buf strings.Builder
	buf.WriteRune('"')
	previous := '"'
	for r.Len() > 0 {
		c, _, err := r.ReadRune()
		if err != nil {
			panic("Lexing error while eating string")
		}
		buf.WriteRune(c)
		if c == '"' && previous != '\\' {
			break
		}
		previous = c
	}
	return buf.String()
}

func eatSymbol(r *strings.Reader) string {
	var buf strings.Builder
	var previous rune

	for r.Len() > 0 {
		c, _, err := r.ReadRune()
		if err != nil {
			panic("Lexing error while eating symbol")
		}
		if c == ')' {
			r.UnreadRune()
			return buf.String()
		}
		if unicode.IsSpace(c) && previous != '\\' {
			break
		}
		buf.WriteRune(c)
		previous = c
	}
	return buf.String()
}

func eatQuasiQuote(r *strings.Reader) []string {
	tokens := make([]string, 0, 10)
	tokens = append(tokens, "(", "quote")
	list := false
	depth := 0
	var currentString string

	Loop:
	for r.Len() > 0 {
		c, _, err := r.ReadRune()
		if err != nil {
			break Loop
		}

		if unicode.IsSpace(c) && !list {
			panic("Tokenization error: invalid construct '` '")
		}

		switch c {
		case '(':
			list = true
			depth++
			tokens = append(tokens, string(c))
		case ')':
			depth--
			tokens = append(tokens, string(c))
			if depth == 0 {
				break Loop
			}
		case '`':
			t := eatQuasiQuote(r)
			tokens = append(tokens, t...)
		case '"':
			currentString = eatString(r)
			tokens = append(tokens, currentString)
			if !list {
				break Loop
			}
		default:
			r.UnreadRune()
			currentString = eatSymbol(r)
			if currentString != "" {
				tokens = append(tokens, currentString)
			}
			if !list {
				break Loop
			}
		}
	}
	tokens = append(tokens, ")")
	return tokens
}

func Tokenize(s string) []string {
	reader := strings.NewReader(s)
	tokens := make([]string, 0)
	var currentString string
	TokenizationLoop:
	for ;reader.Len() > 0; {
		c, _, err := reader.ReadRune()
		if err != nil {
			break TokenizationLoop
		}

		if unicode.IsSpace(c) {
			eatWhiteSpace(reader)
			continue
		}

		switch c {
		case '(', ')':
			tokens = append(tokens, string(c))
		case '`':
			t := eatQuasiQuote(reader)
			tokens = append(tokens, t...)
		case '"':
			currentString = eatString(reader)
			tokens = append(tokens, currentString)
		case ';':
			eatSingleLineComment(reader)
		case '#':
			c, _, err := reader.ReadRune()
			if err != nil {
				break TokenizationLoop
			}
			tokens = append(tokens, fmt.Sprintf("#%c", c))
		default:
			reader.UnreadRune()
			currentString = eatSymbol(reader)
			if currentString != "" {
				tokens = append(tokens, currentString)
			}
		}
	}


	return tokens
}