Adds lexer and beginnings of parser

This commit is contained in:
Brian Evans 2019-07-16 11:41:54 -07:00
parent 4fc28a42bd
commit dbe94ee680
2 changed files with 157 additions and 0 deletions

157
parser/lexer.go Normal file
View File

@ -0,0 +1,157 @@
package parser
import (
"bufio"
"bytes"
"io"
"strings"
)
type Token struct {
kind int
val string
}
type scanner struct {
r *bufio.Reader
}
var eof rune = rune(0)
const (
Number int = iota
Text
Opperator
Whitespace
Comment
Newline
End
Illegal
)
// - ~ + - ~ + //
// Receivers //
// - ~ + - ~ + //
func (s *scanner) read() rune {
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
return ch
}
func (s *scanner) unread() {
_ = s.r.UnreadRune()
}
func (s *scanner) scan() Token {
char := s.read()
if isWhitespace(char) {
s.unread()
return s.scanWhitespace()
} else if isLetter(char) {
s.unread()
return s.scanText()
} else if isNumber(char) {
s.unread()
return s.scanNumber()
} else if char == '#' {
s.unread()
return s.scanComment()
} else if isOpperator() {
return Token{Opperator, string(char)}
} else if char == '\n' {
return Token{Newline,""}
} else if char == eof {
return Token{End, ""}
}
return Token{illegal, string(char)}
}
func (s *scanner) scanWhitespace() Token {
var buf bytes.Buffer
buf.WriteRune(s.read())
for {
if ch := s.read(); ch == eof {
s.unread()
break
} else if !isWhitespace(ch) {
s.unread()
break
} else {
_, _ = buf.WriteRune(ch)
}
}
return Token{Whitespace, buf.String()}
}
func (s *scanner) scanText() Token {
var buf bytes.Buffer
buf.WriteRune(s.read())
for {
if ch := s.read(); ch == eof {
s.unread()
break
} else if !isLetter(ch) {
s.unread()
break
} else {
_, _ = buf.WriteRune(ch)
}
return Token{Text, buf.String()}
}
}
func (s *scanner) scanComment() {
var buf bytes.Buffer
buf.WriteRune(s.read())
for {
if ch := s.read(); ch == eof || ch == '\n' {
s.unread()
break
} else {
_, _ = buf.WriteRune(ch)
}
}
return Token{Comment, buf.String()}
}
// - ~ + - ~ + //
// Functions //
// - ~ + - ~ + //
func isWhitespace(ch rune) bool {
return ch == ' ' || ch == '\t' || ch == '\r'
}
func isLetter(ch rune) bool {
return ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'
}
func isNumber(ch rune) bool {
return ch >= '0' && ch <= '9'
}
func isOpperator(ch rune) bool {
return ch == '=' || ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '^' || ch == '%'
}
func NewScanner(r io.Reader) *scanner {
return &scanner{r: bufio.NewReader(r)}
}

0
parser/parser.go Normal file
View File