https://github.com/akkartik/mu/blob/main/shell/parenthesize.mu

      1 ## insert explicit parens based on indentation
      2
      3 # Design goals:
      4 #  keywords in other languages should look different from functions: def, if, while, etc.
      5 #  fully-parenthesized expressions should not be messed with
      6 #    ignore indent when lines start with parens
      7 #    ignore indent inside parens
      8 #    no modes to disable this pass
      9 #  introduce no new operators
     10 #    the language doesn't use nested lists like Scheme's `cond`
     11 #    lines with one word are never wrapped in parens
     12 #  encourage macros to explicitly insert all parens
     13 #    ignore indent inside backquote
     14
     15 fn parenthesize in: (addr stream token), out: (addr stream token), trace: (addr trace) {
     16   trace-text trace, "parenthesize", "insert parens"
     17   trace-lower trace
     18   var buffer-storage: (stream token 0x40)
     19   var buffer/edi: (addr stream token) <- address buffer-storage
     20   var curr-line-indent: int
     21   var num-words-in-line: int
     22   var paren-at-start-of-line?: boolean
     23   var explicit-open-parens-storage: int
     24   var explicit-open-parens/ebx: (addr int) <- address explicit-open-parens-storage
     25   var implicit-open-parens-storage: int-stack
     26   var implicit-open-parens/esi: (addr int-stack) <- address implicit-open-parens-storage
     27   initialize-int-stack implicit-open-parens, 0x10  # potentially a major memory leak
     28   rewind-stream in
     29   {
     30     var done?/eax: boolean <- stream-empty? in
     31     compare done?, 0/false
     32     break-if-!=
     33     #
     34     var curr-token-storage: token
     35     var curr-token/ecx: (addr token) <- address curr-token-storage
     36     read-from-stream in, curr-token
     37 #?     dump-token-from-cursor curr-token
     38     # update state
     39     {
     40       var is-indent?/eax: boolean <- indent-token? curr-token
     41       compare is-indent?, 0/false
     42       break-if-=
     43       copy-to num-words-in-line, 0
     44       copy-to paren-at-start-of-line?, 0/false
     45       var tmp/eax: int <- indent-level curr-token
     46       copy-to curr-line-indent, tmp
     47     }
     48     {
     49       var is-word?/eax: boolean <- word-token? curr-token
     50       compare is-word?, 0/false
     51       break-if-=
     52       increment num-words-in-line
     53     }
     54     {
     55       compare num-words-in-line, 0
     56       break-if-!=
     57       var is-open?/eax: boolean <- open-paren-token? curr-token
     58       compare is-open?, 0/false
     59       break-if-=
     60       copy-to paren-at-start-of-line?, 1/true
     61     }
     62     #
     63     $parenthesize:emit: {
     64       {
     65         compare paren-at-start-of-line?, 0/false
     66         break-if-=
     67 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "A", 7/fg 0/bg
     68         emit-all buffer, curr-token, out, explicit-open-parens
     69         break $parenthesize:emit
     70       }
     71       {
     72         var is-indent?/eax: boolean <- indent-token? curr-token
     73         compare is-indent?, 0/false
     74         break-if-=
     75 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "B", 7/fg 0/bg
     76         emit-all buffer, curr-token, out, explicit-open-parens
     77         break $parenthesize:emit
     78       }
     79       {
     80         compare num-words-in-line, 2
     81         break-if->=
     82 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "C", 7/fg 0/bg
     83         write-to-stream buffer, curr-token
     84         break $parenthesize:emit
     85       }
     86       {
     87         compare num-words-in-line, 2
     88         break-if-!=
     89         var is-word?/eax: boolean <- word-token? curr-token
     90         compare is-word?, 0/false
     91         break-if-=
     92         compare *explicit-open-parens, 0
     93         break-if-!=
     94 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "(\n", 7/fg 0/bg
     95         var paren-storage: token
     96         var paren-token/eax: (addr token) <- address paren-storage
     97         initialize-token paren-token, "("
     98         write-to-stream out, paren-token
     99         push-int-stack implicit-open-parens, curr-line-indent
    100       }
    101       emit-all buffer, curr-token, out, explicit-open-parens
    102     }
    103     {
    104       var is-indent?/eax: boolean <- indent-token? curr-token
    105       compare is-indent?, 0/false
    106       break-if-=
    107       {
    108         # . loop check
    109         var done?/eax: boolean <- int-stack-empty? implicit-open-parens
    110         compare done?, 0/false
    111         break-if-!=
    112         var top-indent/eax: int <- int-stack-top implicit-open-parens
    113         compare top-indent, curr-line-indent
    114         break-if-<
    115         # . loop body
    116         var paren-storage: token
    117         var paren-token/eax: (addr token) <- address paren-storage
    118         initialize-token paren-token, ")"
    119         write-to-stream out, paren-token
    120         # . update
    121         var dummy/eax: int <- pop-int-stack implicit-open-parens
    122         loop
    123       }
    124     }
    125     loop
    126   }
    127   emit-all buffer, 0/no-curr-token, out, explicit-open-parens
    128   {
    129     # . loop check
    130     var done?/eax: boolean <- int-stack-empty? implicit-open-parens
    131     compare done?, 0/false
    132     break-if-!=
    133     # . loop body
    134     var paren-storage: token
    135     var paren-token/eax: (addr token) <- address paren-storage
    136     initialize-token paren-token, ")"
    137     write-to-stream out, paren-token
    138     # . update
    139     var dummy/eax: int <- pop-int-stack implicit-open-parens
    140     loop
    141   }
    142   trace-higher trace
    143 }
    144
    145 fn indent-level _in: (addr token) -> _/eax: int {
    146   var in/eax: (addr token) <- copy _in
    147   var result/eax: (addr int) <- get in, number-data
    148   return *result
    149 }
    150
    151 fn word-token? in: (addr token) -> _/eax: boolean {
    152   {
    153     var is-indent?/eax: boolean <- indent-token? in
    154     compare is-indent?, 0/false
    155     break-if-!=
    156     var is-bracket?/eax: boolean <- bracket-token? in  # overzealously checks for [], but shouldn't ever encounter it
    157     compare is-bracket?, 0/false
    158     break-if-!=
    159     var is-quote?/eax: boolean <- quote-token? in
    160     compare is-quote?, 0/false
    161     break-if-!=
    162     var is-backquote?/eax: boolean <- backquote-token? in
    163     compare is-backquote?, 0/false
    164     break-if-!=
    165     var is-unquote?/eax: boolean <- unquote-token? in
    166     compare is-unquote?, 0/false
    167     break-if-!=
    168     var is-unquote-splice?/eax: boolean <- unquote-splice-token? in
    169     compare is-unquote-splice?, 0/false
    170     break-if-!=
    171     return 1/true
    172   }
    173   return 0/false
    174 }
    175
    176 fn emit-all first: (addr stream token), second: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
    177   rewind-stream first
    178   {
    179     var done?/eax: boolean <- stream-empty? first
    180     compare done?, 0/false
    181     break-if-!=
    182     var curr-token-storage: token
    183     var curr-token/eax: (addr token) <- address curr-token-storage
    184     read-from-stream first, curr-token
    185     emit curr-token, out, explicit-open-parens
    186     loop
    187   }
    188   clear-stream first
    189   {
    190     compare second, 0
    191     break-if-=
    192     emit second, out, explicit-open-parens
    193   }
    194 }
    195
    196 fn emit t: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
    197   {
    198     var is-indent?/eax: boolean <- indent-token? t
    199     compare is-indent?, 0/false
    200     break-if-=
    201     return
    202   }
    203   write-to-stream out, t
    204   var explicit-open-parens/edi: (addr int) <- copy explicit-open-parens
    205   {
    206     var is-open?/eax: boolean <- open-paren-token? t
    207     compare is-open?, 0/false
    208     break-if-=
    209     increment *explicit-open-parens
    210   }
    211   {
    212     var is-close?/eax: boolean <- close-paren-token? t
    213     compare is-close?, 0/false
    214     break-if-=
    215     decrement *explicit-open-parens
    216     compare *explicit-open-parens, 0
    217     break-if->=
    218     abort "emit: extra ')'"
    219   }
    220 }
    221
    222 # helper for checking parenthesize
    223 fn emit-salient-tokens in: (addr stream token), out: (addr stream token) {
    224   rewind-stream in
    225   {
    226     var done?/eax: boolean <- stream-empty? in
    227     compare done?, 0/false
    228     break-if-!=
    229     var token-storage: token
    230     var token/edx: (addr token) <- address token-storage
    231     read-from-stream in, token
    232     # skip tokens should be skipped
    233     var is-skip?/eax: boolean <- skip-token? token
    234     compare is-skip?, 0/false
    235     loop-if-!=
    236     # indent tokens should be skipped
    237     var is-indent?/eax: boolean <- indent-token? token
    238     compare is-indent?, 0/false
    239     loop-if-!=
    240     #
    241     write-to-stream out, token  # shallow copy
    242     loop
    243   }
    244 }
    245
    246 fn test-parenthesize {
    247   check-parenthesize "a b c  ", "(a b c)", "F - test-parenthesize/1"
    248   check-parenthesize "a (b)", "(a (b))", "F - test-parenthesize/2"
    249   check-parenthesize "a (b c)", "(a (b c))", "F - test-parenthesize/3"
    250   check-parenthesize "a (b c) d", "(a (b c) d)", "F - test-parenthesize/4"
    251   check-parenthesize "a b c\nd ef", "(a b c) (d ef)", "F - test-parenthesize/5-multiple-lines"
    252   check-parenthesize "a b c\n  d ef", "(a b c (d ef))", "F - test-parenthesize/6-indented"
    253   check-parenthesize "a b c\n  (d ef)", "(a b c (d ef))", "F - test-parenthesize/7-indented"
    254   check-parenthesize "a b c\n  (d ef)\n  g", "(a b c (d ef) g)", "F - test-parenthesize/8-indented"
    255   check-parenthesize "a b c\n  d e\n    f\ny", "(a b c (d e f)) y", "F - test-parenthesize/9-indented"
    256   check-parenthesize "#a\na b", "(a b)", "F - test-parenthesize/10-initial-comment"
    257 #? a b c
    258 #?     d ef
    259 #?
    260 #?   g
    261 #?   check-parenthesize "a b c\n    d ef\n\n  g", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
    262 #?   check-parenthesize "a b c\n    d ef\n\n  g #abc", "(a b c (d ef)) g", "F - test-parenthesize/11-comments"
    263   check-parenthesize "a b c\n    d ef\n\n  g #abc", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
    264 #? a b c
    265 #?   '(d ef)
    266 #?
    267 #?   g #abc
    268 #?   check-parenthesize "a b c\n  '(d ef)\n  g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
    269   check-parenthesize "a b c\n  '(d ef)\n\n  g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
    270   check-parenthesize "  a b c", "(a b c)", "F - test-parenthesize/13-initial-indent"
    271   check-parenthesize "    a b c\n  34", "(a b c) 34", "F - test-parenthesize/14-initial-indent"
    272   check-parenthesize "def foo\n    a b c\n  d e\nnewdef", "(def foo (a b c) (d e)) newdef", "F - test-parenthesize/14"
    273   check-parenthesize "  a a\n    a\ny", "(a a a) y", "F - test-parenthesize/15-group-before-too-much-outdent"
    274   check-parenthesize "a `(b c)", "(a `(b c))", "F - test-parenthesize/16-backquote"
    275   check-parenthesize "'a b c", "('a b c)", "F - test-parenthesize/17-quote"
    276   check-parenthesize ",a b c", "(,a b c)", "F - test-parenthesize/18-unquote"
    277   check-parenthesize ",@a b c", "(,@a b c)", "F - test-parenthesize/19-unquote-splice"
    278   check-parenthesize "a b\n  'c\n  ,d\n  e", "(a b 'c ,d e)", "F - test-parenthesize/20-quotes-are-not-words"
    279   check-parenthesize "def foo\n#a b c\n  d e\nnew", "(def foo (d e)) new", "F - test-parenthesize/21-group-across-comments"
    280 }
    281
    282 fn test-parenthesize-skips-lines-with-initial-parens {
    283   check-parenthesize "(a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/1"
    284   check-parenthesize "(a (b c))", "(a (b c))", "F - test-parenthesize-skips-lines-with-initial-parens/2"
    285   check-parenthesize "(a () b)", "(a () b)", "F - test-parenthesize-skips-lines-with-initial-parens/3"
    286   check-parenthesize "  (a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/initial-indent"
    287   check-parenthesize "(a b c\n  bc\n    def\n  gh)", "(a b c bc def gh)", "F - test-parenthesize-skips-lines-with-initial-parens/outdent"
    288   check-parenthesize "(a b c\n  (def gh)\n    (i j k)\n  lm\n\n\n    (no p))", "(a b c (def gh) (i j k) lm (no p))", "F - test-parenthesize-skips-lines-with-initial-parens/fully-parenthesized"
    289   check-parenthesize ",(a b c)", ",(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote"
    290   check-parenthesize ",@(a b c)", ",@(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote-splice"
    291   check-parenthesize ",,(a b c)", ",,(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-nested-unquote"
    292   check-parenthesize "(def foo\n    #a b c\n  d e)\nnew", "(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment"
    293   check-parenthesize "`(def foo\n    #a b c\n  d e)\nnew", "`(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment-after-backquote"
    294   check-parenthesize "  (a b c\n    d e)", "(a b c d e)", "F - test-parenthesize-skips-lines-with-initial-parens/with-indent"
    295   check-parenthesize "def foo(a (b)\n    c d)\n  d e\nnew", "(def foo (a (b) c d) (d e)) new", "F - test-parenthesize-skips-lines-with-initial-parens/inside-arg-lists"
    296 }
    297
    298 fn test-parenthesize-skips-single-word-lines {
    299   # lines usually get grouped with later indented lines
    300   check-parenthesize "a b\n  c", "(a b c)", "F - test-parenthesize-skips-single-word-lines/0"
    301   # but single-word lines don't
    302   check-parenthesize "a\n  c", "a c", "F - test-parenthesize-skips-single-word-lines/1"
    303   check-parenthesize "a", "a", "F - test-parenthesize-skips-single-word-lines/2"
    304   check-parenthesize "a  \nb\nc", "a b c", "F - test-parenthesize-skips-single-word-lines/3"
    305 }
    306
    307 fn check-parenthesize actual: (addr array byte), expected: (addr array byte), message: (addr array byte) {
    308   var trace-storage: trace
    309   var trace/edx: (addr trace) <- address trace-storage
    310   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
    311   #
    312   var actual-buffer-storage: gap-buffer
    313   var actual-buffer/eax: (addr gap-buffer) <- address actual-buffer-storage
    314   initialize-gap-buffer-with actual-buffer, actual
    315   var actual-tokens-storage: (stream token 0x40)
    316   var actual-tokens/esi: (addr stream token) <- address actual-tokens-storage
    317   tokenize-and-parenthesize actual-buffer, actual-tokens, trace
    318   #
    319   var expected-buffer-storage: gap-buffer
    320   var expected-buffer/eax: (addr gap-buffer) <- address expected-buffer-storage
    321   initialize-gap-buffer-with expected-buffer, expected
    322   var expected-tokens-storage: (stream token 0x40)
    323   var expected-tokens/edi: (addr stream token) <- address expected-tokens-storage
    324   tokenize-salient expected-buffer, expected-tokens, trace
    325   #
    326   rewind-stream actual-tokens
    327   check-token-streams-data-equal actual-tokens, expected-tokens, message
    328 }
    329
    330 fn check-token-streams-data-equal actual: (addr stream token), expected: (addr stream token), message: (addr array byte) {
    331   rewind-stream actual
    332   rewind-stream expected
    333   {
    334     # loop termination checks
    335     var actual-done?/eax: boolean <- stream-empty? actual
    336     {
    337       compare actual-done?, 0/false
    338       break-if-=
    339       var expected-done?/eax: boolean <- stream-empty? expected
    340       compare expected-done?, 0/false
    341       {
    342         break-if-!=
    343         # actual empty, but expected not empty
    344         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
    345         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too short\n", 3/fg=cyan 0/bg
    346         count-test-failure
    347         return
    348       }
    349       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ".", 3/fg/cyan, 0/bg
    350       return
    351     }
    352     var expected-done?/eax: boolean <- stream-empty? expected
    353     compare expected-done?, 0/false
    354     {
    355       break-if-=
    356       # actual not empty, but expected empty
    357       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
    358       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too long\n", 3/fg=cyan 0/bg
    359       count-test-failure
    360       return
    361     }
    362     # loop body
    363     var curr-token-storage: token
    364     var curr-token/ecx: (addr token) <- address curr-token-storage
    365     read-from-stream actual, curr-token
    366 #?     dump-token-from-cursor curr-token
    367     var expected-token-storage: token
    368     var expected-token/edx: (addr token) <- address expected-token-storage
    369     read-from-stream expected, expected-token
    370 #?     dump-token-from-cursor expected-token
    371     var match?/eax: boolean <- tokens-equal? curr-token, expected-token
    372     compare match?, 0/false
    373     {
    374       break-if-!=
    375       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
    376       count-test-failure
    377       return
    378     }
    379     loop
    380   }
    381 }
    382
    383 fn tokenize-and-parenthesize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
    384   var tokens-storage: (stream token 0x400)
    385   var tokens/edx: (addr stream token) <- address tokens-storage
    386   tokenize in, tokens, trace
    387   var error?/eax: boolean <- has-errors? trace
    388   compare error?, 0/false
    389   {
    390     break-if-=
    391     return
    392   }
    393   parenthesize tokens, out, trace
    394 }
    395
    396 fn tokenize-salient in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
    397   var tokens-storage: (stream token 0x400)
    398   var tokens/edx: (addr stream token) <- address tokens-storage
    399   tokenize in, tokens, trace
    400   var error?/eax: boolean <- has-errors? trace
    401   compare error?, 0/false
    402   {
    403     break-if-=
    404     return
    405   }
    406   emit-salient-tokens tokens, out
    407 }