408 lines
16 KiB
Forth
408 lines
16 KiB
Forth
## insert explicit parens based on indentation
|
|
|
|
# Design goals:
|
|
# keywords in other languages should look different from functions: def, if, while, etc.
|
|
# fully-parenthesized expressions should not be messed with
|
|
# ignore indent when lines start with parens
|
|
# ignore indent inside parens
|
|
# no modes to disable this pass
|
|
# introduce no new operators
|
|
# the language doesn't use nested lists like Scheme's `cond`
|
|
# lines with one word are never wrapped in parens
|
|
# encourage macros to explicitly insert all parens
|
|
# ignore indent inside backquote
|
|
|
|
fn parenthesize in: (addr stream token), out: (addr stream token), trace: (addr trace) {
|
|
trace-text trace, "parenthesize", "insert parens"
|
|
trace-lower trace
|
|
var buffer-storage: (stream token 0x40)
|
|
var buffer/edi: (addr stream token) <- address buffer-storage
|
|
var curr-line-indent: int
|
|
var num-words-in-line: int
|
|
var paren-at-start-of-line?: boolean
|
|
var explicit-open-parens-storage: int
|
|
var explicit-open-parens/ebx: (addr int) <- address explicit-open-parens-storage
|
|
var implicit-open-parens-storage: int-stack
|
|
var implicit-open-parens/esi: (addr int-stack) <- address implicit-open-parens-storage
|
|
initialize-int-stack implicit-open-parens, 0x10 # potentially a major memory leak
|
|
rewind-stream in
|
|
{
|
|
var done?/eax: boolean <- stream-empty? in
|
|
compare done?, 0/false
|
|
break-if-!=
|
|
#
|
|
var curr-token-storage: token
|
|
var curr-token/ecx: (addr token) <- address curr-token-storage
|
|
read-from-stream in, curr-token
|
|
#? dump-token-from-cursor curr-token
|
|
# update state
|
|
{
|
|
var is-indent?/eax: boolean <- indent-token? curr-token
|
|
compare is-indent?, 0/false
|
|
break-if-=
|
|
copy-to num-words-in-line, 0
|
|
copy-to paren-at-start-of-line?, 0/false
|
|
var tmp/eax: int <- indent-level curr-token
|
|
copy-to curr-line-indent, tmp
|
|
}
|
|
{
|
|
var is-word?/eax: boolean <- word-token? curr-token
|
|
compare is-word?, 0/false
|
|
break-if-=
|
|
increment num-words-in-line
|
|
}
|
|
{
|
|
compare num-words-in-line, 0
|
|
break-if-!=
|
|
var is-open?/eax: boolean <- open-paren-token? curr-token
|
|
compare is-open?, 0/false
|
|
break-if-=
|
|
copy-to paren-at-start-of-line?, 1/true
|
|
}
|
|
#
|
|
$parenthesize:emit: {
|
|
{
|
|
compare paren-at-start-of-line?, 0/false
|
|
break-if-=
|
|
#? draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "A", 7/fg 0/bg
|
|
emit-all buffer, curr-token, out, explicit-open-parens
|
|
break $parenthesize:emit
|
|
}
|
|
{
|
|
var is-indent?/eax: boolean <- indent-token? curr-token
|
|
compare is-indent?, 0/false
|
|
break-if-=
|
|
#? draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "B", 7/fg 0/bg
|
|
emit-all buffer, curr-token, out, explicit-open-parens
|
|
break $parenthesize:emit
|
|
}
|
|
{
|
|
compare num-words-in-line, 2
|
|
break-if->=
|
|
#? draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "C", 7/fg 0/bg
|
|
write-to-stream buffer, curr-token
|
|
break $parenthesize:emit
|
|
}
|
|
{
|
|
compare num-words-in-line, 2
|
|
break-if-!=
|
|
var is-word?/eax: boolean <- word-token? curr-token
|
|
compare is-word?, 0/false
|
|
break-if-=
|
|
compare *explicit-open-parens, 0
|
|
break-if-!=
|
|
#? draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "(\n", 7/fg 0/bg
|
|
var paren-storage: token
|
|
var paren-token/eax: (addr token) <- address paren-storage
|
|
initialize-token paren-token, "("
|
|
write-to-stream out, paren-token
|
|
push-int-stack implicit-open-parens, curr-line-indent
|
|
}
|
|
emit-all buffer, curr-token, out, explicit-open-parens
|
|
}
|
|
{
|
|
var is-indent?/eax: boolean <- indent-token? curr-token
|
|
compare is-indent?, 0/false
|
|
break-if-=
|
|
{
|
|
# . loop check
|
|
var done?/eax: boolean <- int-stack-empty? implicit-open-parens
|
|
compare done?, 0/false
|
|
break-if-!=
|
|
var top-indent/eax: int <- int-stack-top implicit-open-parens
|
|
compare top-indent, curr-line-indent
|
|
break-if-<
|
|
# . loop body
|
|
var paren-storage: token
|
|
var paren-token/eax: (addr token) <- address paren-storage
|
|
initialize-token paren-token, ")"
|
|
write-to-stream out, paren-token
|
|
# . update
|
|
var dummy/eax: int <- pop-int-stack implicit-open-parens
|
|
loop
|
|
}
|
|
}
|
|
loop
|
|
}
|
|
emit-all buffer, 0/no-curr-token, out, explicit-open-parens
|
|
{
|
|
# . loop check
|
|
var done?/eax: boolean <- int-stack-empty? implicit-open-parens
|
|
compare done?, 0/false
|
|
break-if-!=
|
|
# . loop body
|
|
var paren-storage: token
|
|
var paren-token/eax: (addr token) <- address paren-storage
|
|
initialize-token paren-token, ")"
|
|
write-to-stream out, paren-token
|
|
# . update
|
|
var dummy/eax: int <- pop-int-stack implicit-open-parens
|
|
loop
|
|
}
|
|
trace-higher trace
|
|
}
|
|
|
|
fn indent-level _in: (addr token) -> _/eax: int {
|
|
var in/eax: (addr token) <- copy _in
|
|
var result/eax: (addr int) <- get in, number-data
|
|
return *result
|
|
}
|
|
|
|
fn word-token? in: (addr token) -> _/eax: boolean {
|
|
{
|
|
var is-indent?/eax: boolean <- indent-token? in
|
|
compare is-indent?, 0/false
|
|
break-if-!=
|
|
var is-bracket?/eax: boolean <- bracket-token? in # overzealously checks for [], but shouldn't ever encounter it
|
|
compare is-bracket?, 0/false
|
|
break-if-!=
|
|
var is-quote?/eax: boolean <- quote-token? in
|
|
compare is-quote?, 0/false
|
|
break-if-!=
|
|
var is-backquote?/eax: boolean <- backquote-token? in
|
|
compare is-backquote?, 0/false
|
|
break-if-!=
|
|
var is-unquote?/eax: boolean <- unquote-token? in
|
|
compare is-unquote?, 0/false
|
|
break-if-!=
|
|
var is-unquote-splice?/eax: boolean <- unquote-splice-token? in
|
|
compare is-unquote-splice?, 0/false
|
|
break-if-!=
|
|
return 1/true
|
|
}
|
|
return 0/false
|
|
}
|
|
|
|
fn emit-all first: (addr stream token), second: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
|
|
rewind-stream first
|
|
{
|
|
var done?/eax: boolean <- stream-empty? first
|
|
compare done?, 0/false
|
|
break-if-!=
|
|
var curr-token-storage: token
|
|
var curr-token/eax: (addr token) <- address curr-token-storage
|
|
read-from-stream first, curr-token
|
|
emit curr-token, out, explicit-open-parens
|
|
loop
|
|
}
|
|
clear-stream first
|
|
{
|
|
compare second, 0
|
|
break-if-=
|
|
emit second, out, explicit-open-parens
|
|
}
|
|
}
|
|
|
|
fn emit t: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
|
|
{
|
|
var is-indent?/eax: boolean <- indent-token? t
|
|
compare is-indent?, 0/false
|
|
break-if-=
|
|
return
|
|
}
|
|
write-to-stream out, t
|
|
var explicit-open-parens/edi: (addr int) <- copy explicit-open-parens
|
|
{
|
|
var is-open?/eax: boolean <- open-paren-token? t
|
|
compare is-open?, 0/false
|
|
break-if-=
|
|
increment *explicit-open-parens
|
|
}
|
|
{
|
|
var is-close?/eax: boolean <- close-paren-token? t
|
|
compare is-close?, 0/false
|
|
break-if-=
|
|
decrement *explicit-open-parens
|
|
compare *explicit-open-parens, 0
|
|
break-if->=
|
|
abort "emit: extra ')'"
|
|
}
|
|
}
|
|
|
|
# helper for checking parenthesize
|
|
fn emit-salient-tokens in: (addr stream token), out: (addr stream token) {
|
|
rewind-stream in
|
|
{
|
|
var done?/eax: boolean <- stream-empty? in
|
|
compare done?, 0/false
|
|
break-if-!=
|
|
var token-storage: token
|
|
var token/edx: (addr token) <- address token-storage
|
|
read-from-stream in, token
|
|
# skip tokens should be skipped
|
|
var is-skip?/eax: boolean <- skip-token? token
|
|
compare is-skip?, 0/false
|
|
loop-if-!=
|
|
# indent tokens should be skipped
|
|
var is-indent?/eax: boolean <- indent-token? token
|
|
compare is-indent?, 0/false
|
|
loop-if-!=
|
|
#
|
|
write-to-stream out, token # shallow copy
|
|
loop
|
|
}
|
|
}
|
|
|
|
fn test-parenthesize {
|
|
check-parenthesize "a b c ", "(a b c)", "F - test-parenthesize/1"
|
|
check-parenthesize "a (b)", "(a (b))", "F - test-parenthesize/2"
|
|
check-parenthesize "a (b c)", "(a (b c))", "F - test-parenthesize/3"
|
|
check-parenthesize "a (b c) d", "(a (b c) d)", "F - test-parenthesize/4"
|
|
check-parenthesize "a b c\nd ef", "(a b c) (d ef)", "F - test-parenthesize/5-multiple-lines"
|
|
check-parenthesize "a b c\n d ef", "(a b c (d ef))", "F - test-parenthesize/6-indented"
|
|
check-parenthesize "a b c\n (d ef)", "(a b c (d ef))", "F - test-parenthesize/7-indented"
|
|
check-parenthesize "a b c\n (d ef)\n g", "(a b c (d ef) g)", "F - test-parenthesize/8-indented"
|
|
check-parenthesize "a b c\n d e\n f\ny", "(a b c (d e f)) y", "F - test-parenthesize/9-indented"
|
|
check-parenthesize "#a\na b", "(a b)", "F - test-parenthesize/10-initial-comment"
|
|
#? a b c
|
|
#? d ef
|
|
#?
|
|
#? g
|
|
#? check-parenthesize "a b c\n d ef\n\n g", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
|
|
#? check-parenthesize "a b c\n d ef\n\n g #abc", "(a b c (d ef)) g", "F - test-parenthesize/11-comments"
|
|
check-parenthesize "a b c\n d ef\n\n g #abc", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
|
|
#? a b c
|
|
#? '(d ef)
|
|
#?
|
|
#? g #abc
|
|
#? check-parenthesize "a b c\n '(d ef)\n g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
|
|
check-parenthesize "a b c\n '(d ef)\n\n g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
|
|
check-parenthesize " a b c", "(a b c)", "F - test-parenthesize/13-initial-indent"
|
|
check-parenthesize " a b c\n 34", "(a b c) 34", "F - test-parenthesize/14-initial-indent"
|
|
check-parenthesize "def foo\n a b c\n d e\nnewdef", "(def foo (a b c) (d e)) newdef", "F - test-parenthesize/14"
|
|
check-parenthesize " a a\n a\ny", "(a a a) y", "F - test-parenthesize/15-group-before-too-much-outdent"
|
|
check-parenthesize "a `(b c)", "(a `(b c))", "F - test-parenthesize/16-backquote"
|
|
check-parenthesize "'a b c", "('a b c)", "F - test-parenthesize/17-quote"
|
|
check-parenthesize ",a b c", "(,a b c)", "F - test-parenthesize/18-unquote"
|
|
check-parenthesize ",@a b c", "(,@a b c)", "F - test-parenthesize/19-unquote-splice"
|
|
check-parenthesize "a b\n 'c\n ,d\n e", "(a b 'c ,d e)", "F - test-parenthesize/20-quotes-are-not-words"
|
|
check-parenthesize "def foo\n#a b c\n d e\nnew", "(def foo (d e)) new", "F - test-parenthesize/21-group-across-comments"
|
|
}
|
|
|
|
fn test-parenthesize-skips-lines-with-initial-parens {
|
|
check-parenthesize "(a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/1"
|
|
check-parenthesize "(a (b c))", "(a (b c))", "F - test-parenthesize-skips-lines-with-initial-parens/2"
|
|
check-parenthesize "(a () b)", "(a () b)", "F - test-parenthesize-skips-lines-with-initial-parens/3"
|
|
check-parenthesize " (a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/initial-indent"
|
|
check-parenthesize "(a b c\n bc\n def\n gh)", "(a b c bc def gh)", "F - test-parenthesize-skips-lines-with-initial-parens/outdent"
|
|
check-parenthesize "(a b c\n (def gh)\n (i j k)\n lm\n\n\n (no p))", "(a b c (def gh) (i j k) lm (no p))", "F - test-parenthesize-skips-lines-with-initial-parens/fully-parenthesized"
|
|
check-parenthesize ",(a b c)", ",(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote"
|
|
check-parenthesize ",@(a b c)", ",@(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote-splice"
|
|
check-parenthesize ",,(a b c)", ",,(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-nested-unquote"
|
|
check-parenthesize "(def foo\n #a b c\n d e)\nnew", "(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment"
|
|
check-parenthesize "`(def foo\n #a b c\n d e)\nnew", "`(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment-after-backquote"
|
|
check-parenthesize " (a b c\n d e)", "(a b c d e)", "F - test-parenthesize-skips-lines-with-initial-parens/with-indent"
|
|
check-parenthesize "def foo(a (b)\n c d)\n d e\nnew", "(def foo (a (b) c d) (d e)) new", "F - test-parenthesize-skips-lines-with-initial-parens/inside-arg-lists"
|
|
}
|
|
|
|
fn test-parenthesize-skips-single-word-lines {
|
|
# lines usually get grouped with later indented lines
|
|
check-parenthesize "a b\n c", "(a b c)", "F - test-parenthesize-skips-single-word-lines/0"
|
|
# but single-word lines don't
|
|
check-parenthesize "a\n c", "a c", "F - test-parenthesize-skips-single-word-lines/1"
|
|
check-parenthesize "a", "a", "F - test-parenthesize-skips-single-word-lines/2"
|
|
check-parenthesize "a \nb\nc", "a b c", "F - test-parenthesize-skips-single-word-lines/3"
|
|
}
|
|
|
|
fn check-parenthesize actual: (addr array byte), expected: (addr array byte), message: (addr array byte) {
|
|
var trace-storage: trace
|
|
var trace/edx: (addr trace) <- address trace-storage
|
|
initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
|
|
#
|
|
var actual-buffer-storage: gap-buffer
|
|
var actual-buffer/eax: (addr gap-buffer) <- address actual-buffer-storage
|
|
initialize-gap-buffer-with actual-buffer, actual
|
|
var actual-tokens-storage: (stream token 0x40)
|
|
var actual-tokens/esi: (addr stream token) <- address actual-tokens-storage
|
|
tokenize-and-parenthesize actual-buffer, actual-tokens, trace
|
|
#
|
|
var expected-buffer-storage: gap-buffer
|
|
var expected-buffer/eax: (addr gap-buffer) <- address expected-buffer-storage
|
|
initialize-gap-buffer-with expected-buffer, expected
|
|
var expected-tokens-storage: (stream token 0x40)
|
|
var expected-tokens/edi: (addr stream token) <- address expected-tokens-storage
|
|
tokenize-salient expected-buffer, expected-tokens, trace
|
|
#
|
|
rewind-stream actual-tokens
|
|
check-token-streams-data-equal actual-tokens, expected-tokens, message
|
|
}
|
|
|
|
fn check-token-streams-data-equal actual: (addr stream token), expected: (addr stream token), message: (addr array byte) {
|
|
rewind-stream actual
|
|
rewind-stream expected
|
|
{
|
|
# loop termination checks
|
|
var actual-done?/eax: boolean <- stream-empty? actual
|
|
{
|
|
compare actual-done?, 0/false
|
|
break-if-=
|
|
var expected-done?/eax: boolean <- stream-empty? expected
|
|
compare expected-done?, 0/false
|
|
{
|
|
break-if-!=
|
|
# actual empty, but expected not empty
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too short\n", 3/fg=cyan 0/bg
|
|
count-test-failure
|
|
return
|
|
}
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ".", 3/fg/cyan, 0/bg
|
|
return
|
|
}
|
|
var expected-done?/eax: boolean <- stream-empty? expected
|
|
compare expected-done?, 0/false
|
|
{
|
|
break-if-=
|
|
# actual not empty, but expected empty
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too long\n", 3/fg=cyan 0/bg
|
|
count-test-failure
|
|
return
|
|
}
|
|
# loop body
|
|
var curr-token-storage: token
|
|
var curr-token/ecx: (addr token) <- address curr-token-storage
|
|
read-from-stream actual, curr-token
|
|
#? dump-token-from-cursor curr-token
|
|
var expected-token-storage: token
|
|
var expected-token/edx: (addr token) <- address expected-token-storage
|
|
read-from-stream expected, expected-token
|
|
#? dump-token-from-cursor expected-token
|
|
var match?/eax: boolean <- tokens-equal? curr-token, expected-token
|
|
compare match?, 0/false
|
|
{
|
|
break-if-!=
|
|
draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
|
|
count-test-failure
|
|
return
|
|
}
|
|
loop
|
|
}
|
|
}
|
|
|
|
fn tokenize-and-parenthesize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
|
|
var tokens-storage: (stream token 0x400)
|
|
var tokens/edx: (addr stream token) <- address tokens-storage
|
|
tokenize in, tokens, trace
|
|
var error?/eax: boolean <- has-errors? trace
|
|
compare error?, 0/false
|
|
{
|
|
break-if-=
|
|
return
|
|
}
|
|
parenthesize tokens, out, trace
|
|
}
|
|
|
|
fn tokenize-salient in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
|
|
var tokens-storage: (stream token 0x400)
|
|
var tokens/edx: (addr stream token) <- address tokens-storage
|
|
tokenize in, tokens, trace
|
|
var error?/eax: boolean <- has-errors? trace
|
|
compare error?, 0/false
|
|
{
|
|
break-if-=
|
|
return
|
|
}
|
|
emit-salient-tokens tokens, out
|
|
}
|