# Mu - shell/tokenize.mu

https://github.com/akkartik/mu/blob/main/shell/tokenize.mu

      1 # We reuse the cell data structure for tokenization
      2 # Token cells are special, though. They have no type, they're always atoms,
      3 # they always have text-data.
      4
      5 fn tokenize in: (addr gap-buffer), out: (addr stream cell), trace: (addr trace) {
      6   trace-text trace, "tokenize", "tokenize"
      7   trace-lower trace
      8   rewind-gap-buffer in
      9   var token-storage: cell
     10   var token/edx: (addr cell) <- address token-storage
     11   {
     12     skip-whitespace-from-gap-buffer in
     13     var done?/eax: boolean <- gap-buffer-scan-done? in
     14     compare done?, 0/false
     15     break-if-!=
     16     #
     17     next-token in, token, trace
     18     var error?/eax: boolean <- has-errors? trace
     19     compare error?, 0/false
     20     {
     21       break-if-=
     22       return
     23     }
     24     var skip?/eax: boolean <- comment-token? token
     25     compare skip?, 0/false
     26     loop-if-!=
     27     write-to-stream out, token  # shallow-copy text-data
     28     loop
     29   }
     30   trace-higher trace
     31 }
     32
     33 fn test-tokenize-quote {
     34   var in-storage: gap-buffer
     35   var in/esi: (addr gap-buffer) <- address in-storage
     36   initialize-gap-buffer-with in, "'(a)"
     37   #
     38   var stream-storage: (stream cell 0x10)
     39   var stream/edi: (addr stream cell) <- address stream-storage
     40   #
     41   var trace-storage: trace
     42   var trace/edx: (addr trace) <- address trace-storage
     43   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     44   tokenize in, stream, trace
     45   #
     46   var curr-token-storage: cell
     47   var curr-token/ebx: (addr cell) <- address curr-token-storage
     48   read-from-stream stream, curr-token
     49   var quote?/eax: boolean <- quote-token? curr-token
     50   check quote?, "F - test-tokenize-quote: quote"
     51   read-from-stream stream, curr-token
     52   var open-paren?/eax: boolean <- open-paren-token? curr-token
     53   check open-paren?, "F - test-tokenize-quote: open paren"
     54   read-from-stream stream, curr-token  # skip a
     55   read-from-stream stream, curr-token
     56   var close-paren?/eax: boolean <- close-paren-token? curr-token
     57   check close-paren?, "F - test-tokenize-quote: close paren"
     58 }
     59
     60 fn test-tokenize-backquote {
     61   var in-storage: gap-buffer
     62   var in/esi: (addr gap-buffer) <- address in-storage
     63   initialize-gap-buffer-with in, "`(a)"
     64   #
     65   var stream-storage: (stream cell 0x10)
     66   var stream/edi: (addr stream cell) <- address stream-storage
     67   #
     68   var trace-storage: trace
     69   var trace/edx: (addr trace) <- address trace-storage
     70   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     71   tokenize in, stream, trace
     72   #
     73   var curr-token-storage: cell
     74   var curr-token/ebx: (addr cell) <- address curr-token-storage
     75   read-from-stream stream, curr-token
     76   var backquote?/eax: boolean <- backquote-token? curr-token
     77   check backquote?, "F - test-tokenize-backquote: backquote"
     78   read-from-stream stream, curr-token
     79   var open-paren?/eax: boolean <- open-paren-token? curr-token
     80   check open-paren?, "F - test-tokenize-backquote: open paren"
     81   read-from-stream stream, curr-token  # skip a
     82   read-from-stream stream, curr-token
     83   var close-paren?/eax: boolean <- close-paren-token? curr-token
     84   check close-paren?, "F - test-tokenize-backquote: close paren"
     85 }
     86
     87 fn test-tokenize-unquote {
     88   var in-storage: gap-buffer
     89   var in/esi: (addr gap-buffer) <- address in-storage
     90   initialize-gap-buffer-with in, ",(a)"
     91   #
     92   var stream-storage: (stream cell 0x10)
     93   var stream/edi: (addr stream cell) <- address stream-storage
     94   #
     95   var trace-storage: trace
     96   var trace/edx: (addr trace) <- address trace-storage
     97   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     98   tokenize in, stream, trace
     99   #
    100   var curr-token-storage: cell
    101   var curr-token/ebx: (addr cell) <- address curr-token-storage
    102   read-from-stream stream, curr-token
    103   var unquote?/eax: boolean <- unquote-token? curr-token
    104   check unquote?, "F - test-tokenize-unquote: unquote"
    105   read-from-stream stream, curr-token
    106   var open-paren?/eax: boolean <- open-paren-token? curr-token
    107   check open-paren?, "F - test-tokenize-unquote: open paren"
    108   read-from-stream stream, curr-token  # skip a
    109   read-from-stream stream, curr-token
    110   var close-paren?/eax: boolean <- close-paren-token? curr-token
    111   check close-paren?, "F - test-tokenize-unquote: close paren"
    112 }
    113
    114 fn test-tokenize-unquote-splice {
    115   var in-storage: gap-buffer
    116   var in/esi: (addr gap-buffer) <- address in-storage
    117   initialize-gap-buffer-with in, ",@a"
    118   #
    119   var stream-storage: (stream cell 0x10)
    120   var stream/edi: (addr stream cell) <- address stream-storage
    121   #
    122   var trace-storage: trace
    123   var trace/edx: (addr trace) <- address trace-storage
    124   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
    125   tokenize in, stream, trace
    126   #
    127   var curr-token-storage: cell
    128   var curr-token/ebx: (addr cell) <- address curr-token-storage
    129   read-from-stream stream, curr-token
    130   var unquote-splice?/eax: boolean <- unquote-splice-token? curr-token
    131   check unquote-splice?, "F - test-tokenize-unquote-splice: unquote-splice"
    132 }
    133
    134 fn test-tokenize-dotted-list {
    135   var in-storage: gap-buffer
    136   var in/esi: (addr gap-buffer) <- address in-storage
    137   initialize-gap-buffer-with in, "(a . b)"
    138   #
    139   var stream-storage: (stream cell 0x10)
    140   var stream/edi: (addr stream cell) <- address stream-storage
    141   #
    142   var trace-storage: trace
    143   var trace/edx: (addr trace) <- address trace-storage
    144   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
    145   tokenize in, stream, trace
    146   #
    147   var curr-token-storage: cell
    148   var curr-token/ebx: (addr cell) <- address curr-token-storage
    149   read-from-stream stream, curr-token
    150   var open-paren?/eax: boolean <- open-paren-token? curr-token
    151   check open-paren?, "F - test-tokenize-dotted-list: open paren"
    152   read-from-stream stream, curr-token  # skip a
    153   read-from-stream stream, curr-token
    154   var dot?/eax: boolean <- dot-token? curr-token
    155   check dot?, "F - test-tokenize-dotted-list: dot"
    156   read-from-stream stream, curr-token  # skip b
    157   read-from-stream stream, curr-token
    158   var close-paren?/eax: boolean <- close-paren-token? curr-token
    159   check close-paren?, "F - test-tokenize-dotted-list: close paren"
    160 }
    161
    162 fn test-tokenize-stream-literal {
    163   var in-storage: gap-buffer
    164   var in/esi: (addr gap-buffer) <- address in-storage
    165   initialize-gap-buffer-with in, "[abc def]"
    166   #
    167   var stream-storage: (stream cell 0x10)
    168   var stream/edi: (addr stream cell) <- address stream-storage
    169   #
    170   var trace-storage: trace
    171   var trace/edx: (addr trace) <- address trace-storage
    172   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
    173   tokenize in, stream, trace
    174   #
    175   var curr-token-storage: cell
    176   var curr-token/ebx: (addr cell) <- address curr-token-storage
    177   read-from-stream stream, curr-token
    178   var stream?/eax: boolean <- stream-token? curr-token
    179   check stream?, "F - test-tokenize-stream-literal: type"
    180   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
    181   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
    182   var data-equal?/eax: boolean <- stream-data-equal? curr-token-data, "abc def"
    183   check data-equal?, "F - test-tokenize-stream-literal"
    184   var empty?/eax: boolean <- stream-empty? stream
    185   check empty?, "F - test-tokenize-stream-literal: empty?"
    186 }
    187
    188 fn test-tokenize-stream-literal-in-tree {
    189   var in-storage: gap-buffer
    190   var in/esi: (addr gap-buffer) <- address in-storage
    191   initialize-gap-buffer-with in, "([abc def])"
    192   #
    193   var stream-storage: (stream cell 0x10)
    194   var stream/edi: (addr stream cell) <- address stream-storage
    195   #
    196   var trace-storage: trace
    197   var trace/edx: (addr trace) <- address trace-storage
    198   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
    199   tokenize in, stream, trace
    200   #
    201   var curr-token-storage: cell
    202   var curr-token/ebx: (addr cell) <- address curr-token-storage
    203   read-from-stream stream, curr-token
    204   var bracket?/eax: boolean <- bracket-token? curr-token
    205   check bracket?, "F - test-tokenize-stream-literal-in-tree: open paren"
    206   read-from-stream stream, curr-token
    207   var stream?/eax: boolean <- stream-token? curr-token
    208   check stream?, "F - test-tokenize-stream-literal-in-tree: type"
    209   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
    210   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
    211   var data-equal?/eax: boolean <- stream-data-equal? curr-token-data, "abc def"
    212   check data-equal?, "F - test-tokenize-stream-literal-in-tree"
    213   read-from-stream stream, curr-token
    214   var bracket?/eax: boolean <- bracket-token? curr-token
    215   check bracket?, "F - test-tokenize-stream-literal-in-tree: close paren"
    216   var empty?/eax: boolean <- stream-empty? stream
    217   check empty?, "F - test-tokenize-stream-literal-in-tree: empty?"
    218 }
    219
    220 fn next-token in: (addr gap-buffer), _out-cell: (addr cell), trace: (addr trace) {
    221   trace-text trace, "tokenize", "next-token"
    222   trace-lower trace
    223   var _g/eax: grapheme <- peek-from-gap-buffer in
    224   var g/ecx: grapheme <- copy _g
    225   {
    226     var stream-storage: (stream byte 0x40)
    227     var stream/esi: (addr stream byte) <- address stream-storage
    228     write stream, "next: "
    229     var gval/eax: int <- copy g
    230     write-int32-hex stream, gval
    231     trace trace, "tokenize", stream
    232   }
    233   var out-cell/eax: (addr cell) <- copy _out-cell
    234   {
    235     var out-cell-type/eax: (addr int) <- get out-cell, type
    236     copy-to *out-cell-type, 0/uninitialized
    237   }
    238   var out-ah/edi: (addr handle stream byte) <- get out-cell, text-data
    239   $next-token:allocate: {
    240     # Allocate a large buffer if it's a stream.
    241     # Sometimes a whole function definition will need to fit in it.
    242     compare g, 0x5b/open-square-bracket
    243     {
    244       break-if-!=
    245       populate-stream out-ah, 0x400/max-definition-size=1KB
    246       break $next-token:allocate
    247     }
    248     populate-stream out-ah, 0x40
    249   }
    250   var _out/eax: (addr stream byte) <- lookup *out-ah
    251   var out/edi: (addr stream byte) <- copy _out
    252   clear-stream out
    253   $next-token:case: {
    254     # open square brackets begin streams
    255     {
    256       compare g, 0x5b/open-square-bracket
    257       break-if-!=
    258       var dummy/eax: grapheme <- read-from-gap-buffer in  # skip open bracket
    259       next-stream-token in, out, trace
    260       var out-cell/eax: (addr cell) <- copy _out-cell
    261       # streams set the type
    262       var out-cell-type/eax: (addr int) <- get out-cell, type
    263       copy-to *out-cell-type, 3/stream
    264       break $next-token:case
    265     }
    266     # comment
    267     {
    268       compare g, 0x23/comment
    269       break-if-!=
    270       rest-of-line in, out, trace
    271       break $next-token:case
    272     }
    273     # digit
    274     {
    275       var digit?/eax: boolean <- decimal-digit? g
    276       compare digit?, 0/false
    277       break-if-=
    278       next-number-token in, out, trace
    279       break $next-token:case
    280     }
    281     # other symbol char
    282     {
    283       var symbol?/eax: boolean <- symbol-grapheme? g
    284       compare symbol?, 0/false
    285       break-if-=
    286       next-symbol-token in, out, trace
    287       break $next-token:case
    288     }
    289     # unbalanced close square brackets are errors
    290     {
    291       compare g, 0x5d/close-square-bracket
    292       break-if-!=
    293       error trace, "unbalanced ']'"
    294       return
    295     }
    296     # other brackets are always single-char tokens
    297     {
    298       var bracket?/eax: boolean <- bracket-grapheme? g
    299       compare bracket?, 0/false
    300       break-if-=
    301       var g/eax: grapheme <- read-from-gap-buffer in
    302       next-bracket-token g, out, trace
    303       break $next-token:case
    304     }
    305     # non-symbol operators
    306     {
    307       var operator?/eax: boolean <- operator-grapheme? g
    308       compare operator?, 0/false
    309       break-if-=
    310       next-operator-token in, out, trace
    311       break $next-token:case
    312     }
    313     # quote
    314     {
    315       compare g, 0x27/single-quote
    316       break-if-!=
    317       var g/eax: grapheme <- read-from-gap-buffer in  # consume
    318       write-grapheme out, g
    319       break $next-token:case
    320     }
    321     # backquote
    322     {
    323       compare g, 0x60/backquote
    324       break-if-!=
    325       var g/eax: grapheme <- read-from-gap-buffer in  # consume
    326       write-grapheme out, g
    327       break $next-token:case
    328     }
    329     # unquote
    330     {
    331       compare g, 0x2c/comma
    332       break-if-!=
    333       var g/eax: grapheme <- read-from-gap-buffer in  # consume
    334       write-grapheme out, g
    335       # check for unquote-splice
    336       {
    337         var g2/eax: grapheme <- peek-from-gap-buffer in
    338         compare g2, 0x40/at-sign
    339         break-if-!=
    340         g2 <- read-from-gap-buffer in
    341         write-grapheme out, g2
    342       }
    343       break $next-token:case
    344     }
    345     abort "unknown token type"
    346   }
    347   trace-higher trace
    348   var stream-storage: (stream byte 0x400)  # maximum possible token size (next-stream-token)
    349   var stream/eax: (addr stream byte) <- address stream-storage
    350   write stream, "=> "
    351   rewind-stream out
    352   write-stream stream, out
    353   trace trace, "tokenize", stream
    354 }
    355
    356 fn next-symbol-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    357   trace-text trace, "tokenize", "looking for a symbol"
    358   trace-lower trace
    359   $next-symbol-token:loop: {
    360     var done?/eax: boolean <- gap-buffer-scan-done? in
    361     compare done?, 0/false
    362     break-if-!=
    363     var g/eax: grapheme <- peek-from-gap-buffer in
    364     {
    365       var stream-storage: (stream byte 0x40)
    366       var stream/esi: (addr stream byte) <- address stream-storage
    367       write stream, "next: "
    368       var gval/eax: int <- copy g
    369       write-int32-hex stream, gval
    370       trace trace, "tokenize", stream
    371     }
    372     # if non-symbol, return
    373     {
    374       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
    375       compare symbol-grapheme?, 0/false
    376       break-if-!=
    377       trace-text trace, "tokenize", "stop"
    378       break $next-symbol-token:loop
    379     }
    380     var g/eax: grapheme <- read-from-gap-buffer in
    381     write-grapheme out, g
    382     loop
    383   }
    384   trace-higher trace
    385   var stream-storage: (stream byte 0x40)
    386   var stream/esi: (addr stream byte) <- address stream-storage
    387   write stream, "=> "
    388   rewind-stream out
    389   write-stream stream, out
    390   trace trace, "tokenize", stream
    391 }
    392
    393 fn next-operator-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    394   trace-text trace, "tokenize", "looking for a operator"
    395   trace-lower trace
    396   $next-operator-token:loop: {
    397     var done?/eax: boolean <- gap-buffer-scan-done? in
    398     compare done?, 0/false
    399     break-if-!=
    400     var g/eax: grapheme <- peek-from-gap-buffer in
    401     {
    402       var stream-storage: (stream byte 0x40)
    403       var stream/esi: (addr stream byte) <- address stream-storage
    404       write stream, "next: "
    405       var gval/eax: int <- copy g
    406       write-int32-hex stream, gval
    407       trace trace, "tokenize", stream
    408     }
    409     # if non-operator, return
    410     {
    411       var operator-grapheme?/eax: boolean <- operator-grapheme? g
    412       compare operator-grapheme?, 0/false
    413       break-if-!=
    414       trace-text trace, "tokenize", "stop"
    415       break $next-operator-token:loop
    416     }
    417     var g/eax: grapheme <- read-from-gap-buffer in
    418     write-grapheme out, g
    419     loop
    420   }
    421   trace-higher trace
    422   var stream-storage: (stream byte 0x40)
    423   var stream/esi: (addr stream byte) <- address stream-storage
    424   write stream, "=> "
    425   rewind-stream out
    426   write-stream stream, out
    427   trace trace, "tokenize", stream
    428 }
    429
    430 fn next-number-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    431   trace-text trace, "tokenize", "looking for a number"
    432   trace-lower trace
    433   $next-number-token:loop: {
    434     var done?/eax: boolean <- gap-buffer-scan-done? in
    435     compare done?, 0/false
    436     break-if-!=
    437     var g/eax: grapheme <- peek-from-gap-buffer in
    438     {
    439       var stream-storage: (stream byte 0x40)
    440       var stream/esi: (addr stream byte) <- address stream-storage
    441       write stream, "next: "
    442       var gval/eax: int <- copy g
    443       write-int32-hex stream, gval
    444       trace trace, "tokenize", stream
    445     }
    446     # if not symbol grapheme, return
    447     {
    448       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
    449       compare symbol-grapheme?, 0/false
    450       break-if-!=
    451       trace-text trace, "tokenize", "stop"
    452       break $next-number-token:loop
    453     }
    454     # if not digit grapheme, abort
    455     {
    456       var digit?/eax: boolean <- decimal-digit? g
    457       compare digit?, 0/false
    458       break-if-!=
    459       error trace, "invalid number"
    460       return
    461     }
    462     trace-text trace, "tokenize", "append"
    463     var g/eax: grapheme <- read-from-gap-buffer in
    464     write-grapheme out, g
    465     loop
    466   }
    467   trace-higher trace
    468 }
    469
    470 fn next-stream-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    471   trace-text trace, "tokenize", "stream"
    472   {
    473     var empty?/eax: boolean <- gap-buffer-scan-done? in
    474     compare empty?, 0/false
    475     {
    476       break-if-=
    477       error trace, "unbalanced '['"
    478       return
    479     }
    480     var g/eax: grapheme <- read-from-gap-buffer in
    481     compare g, 0x5d/close-square-bracket
    482     break-if-=
    483     write-grapheme out, g
    484     loop
    485   }
    486   var stream-storage: (stream byte 0x400)  # max-definition-size
    487   var stream/esi: (addr stream byte) <- address stream-storage
    488   write stream, "=> "
    489   rewind-stream out
    490   write-stream stream, out
    491   trace trace, "tokenize", stream
    492 }
    493
    494 fn next-bracket-token g: grapheme, out: (addr stream byte), trace: (addr trace) {
    495   trace-text trace, "tokenize", "bracket"
    496   write-grapheme out, g
    497   var stream-storage: (stream byte 0x40)
    498   var stream/esi: (addr stream byte) <- address stream-storage
    499   write stream, "=> "
    500   rewind-stream out
    501   write-stream stream, out
    502   trace trace, "tokenize", stream
    503 }
    504
    505 fn rest-of-line in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    506   trace-text trace, "tokenize", "comment"
    507   {
    508     var empty?/eax: boolean <- gap-buffer-scan-done? in
    509     compare empty?, 0/false
    510     {
    511       break-if-=
    512       return
    513     }
    514     var g/eax: grapheme <- read-from-gap-buffer in
    515     compare g, 0xa/newline
    516     break-if-=
    517     write-grapheme out, g
    518     loop
    519   }
    520   var stream-storage: (stream byte 0x80)
    521   var stream/esi: (addr stream byte) <- address stream-storage
    522   write stream, "=> "
    523   rewind-stream out
    524   write-stream stream, out
    525   trace trace, "tokenize", stream
    526 }
    527
    528 fn symbol-grapheme? g: grapheme -> _/eax: boolean {
    529   ## whitespace
    530   compare g, 9/tab
    531   {
    532     break-if-!=
    533     return 0/false
    534   }
    535   compare g, 0xa/newline
    536   {
    537     break-if-!=
    538     return 0/false
    539   }
    540   compare g, 0x20/space
    541   {
    542     break-if-!=
    543     return 0/false
    544   }
    545   ## quotes
    546   compare g, 0x22/double-quote
    547   {
    548     break-if-!=
    549     return 0/false
    550   }
    551   compare g, 0x60/backquote
    552   {
    553     break-if-!=
    554     return 0/false
    555   }
    556   ## brackets
    557   compare g, 0x28/open-paren
    558   {
    559     break-if-!=
    560     return 0/false
    561   }
    562   compare g, 0x29/close-paren
    563   {
    564     break-if-!=
    565     return 0/false
    566   }
    567   compare g, 0x5b/open-square-bracket
    568   {
    569     break-if-!=
    570     return 0/false
    571   }
    572   compare g, 0x5d/close-square-bracket
    573   {
    574     break-if-!=
    575     return 0/false
    576   }
    577   compare g, 0x7b/open-curly-bracket
    578   {
    579     break-if-!=
    580     return 0/false
    581   }
    582   compare g, 0x7d/close-curly-bracket
    583   {
    584     break-if-!=
    585     return 0/false
    586   }
    587   # - other punctuation
    588   # '!' is a symbol char
    589   compare g, 0x23/hash
    590   {
    591     break-if-!=
    592     return 0/false
    593   }
    594   # '$' is a symbol char
    595   compare g, 0x25/percent
    596   {
    597     break-if-!=
    598     return 0/false
    599   }
    600   compare g, 0x26/ampersand
    601   {
    602     break-if-!=
    603     return 0/false
    604   }
    605   compare g, 0x27/single-quote
    606   {
    607     break-if-!=
    608     return 0/false
    609   }
    610   compare g, 0x60/backquote
    611   {
    612     break-if-!=
    613     return 0/false
    614   }
    615   compare g, 0x2c/comma
    616   {
    617     break-if-!=
    618     return 0/false
    619   }
    620   compare g, 0x40/at-sign
    621   {
    622     break-if-!=
    623     return 0/false
    624   }
    625   compare g, 0x2a/asterisk
    626   {
    627     break-if-!=
    628     return 0/false
    629   }
    630   compare g, 0x2b/plus
    631   {
    632     break-if-!=
    633     return 0/false
    634   }
    635   compare g, 0x2d/dash  # '-' not allowed in symbols
    636   {
    637     break-if-!=
    638     return 0/false
    639   }
    640   compare g, 0x2e/period
    641   {
    642     break-if-!=
    643     return 0/false
    644   }
    645   compare g, 0x2f/slash
    646   {
    647     break-if-!=
    648     return 0/false
    649   }
    650   compare g, 0x3a/colon
    651   {
    652     break-if-!=
    653     return 0/false
    654   }
    655   compare g, 0x3b/semi-colon
    656   {
    657     break-if-!=
    658     return 0/false
    659   }
    660   compare g, 0x3c/less-than
    661   {
    662     break-if-!=
    663     return 0/false
    664   }
    665   compare g, 0x3d/equal
    666   {
    667     break-if-!=
    668     return 0/false
    669   }
    670   compare g, 0x3e/greater-than
    671   {
    672     break-if-!=
    673     return 0/false
    674   }
    675   # '?' is a symbol char
    676   compare g, 0x5c/backslash
    677   {
    678     break-if-!=
    679     return 0/false
    680   }
    681   compare g, 0x5e/caret
    682   {
    683     break-if-!=
    684     return 0/false
    685   }
    686   # '_' is a symbol char
    687   compare g, 0x7c/vertical-line
    688   {
    689     break-if-!=
    690     return 0/false
    691   }
    692   compare g, 0x7e/tilde
    693   {
    694     break-if-!=
    695     return 0/false
    696   }
    697   return 1/true
    698 }
    699
    700 fn bracket-grapheme? g: grapheme -> _/eax: boolean {
    701   compare g, 0x28/open-paren
    702   {
    703     break-if-!=
    704     return 1/true
    705   }
    706   compare g, 0x29/close-paren
    707   {
    708     break-if-!=
    709     return 1/true
    710   }
    711   compare g, 0x5b/open-square-bracket
    712   {
    713     break-if-!=
    714     return 1/true
    715   }
    716   compare g, 0x5d/close-square-bracket
    717   {
    718     break-if-!=
    719     return 1/true
    720   }
    721   compare g, 0x7b/open-curly-bracket
    722   {
    723     break-if-!=
    724     return 1/true
    725   }
    726   compare g, 0x7d/close-curly-bracket
    727   {
    728     break-if-!=
    729     return 1/true
    730   }
    731   return 0/false
    732 }
    733
    734 fn operator-grapheme? g: grapheme -> _/eax: boolean {
    735   # '$' is a symbol char
    736   compare g, 0x25/percent
    737   {
    738     break-if-!=
    739     return 1/false
    740   }
    741   compare g, 0x26/ampersand
    742   {
    743     break-if-!=
    744     return 1/true
    745   }
    746   compare g, 0x27/single-quote
    747   {
    748     break-if-!=
    749     return 0/true
    750   }
    751   compare g, 0x60/backquote
    752   {
    753     break-if-!=
    754     return 0/false
    755   }
    756   compare g, 0x2c/comma
    757   {
    758     break-if-!=
    759     return 0/false
    760   }
    761   compare g, 0x40/at-sign
    762   {
    763     break-if-!=
    764     return 0/false
    765   }
    766   compare g, 0x2a/asterisk
    767   {
    768     break-if-!=
    769     return 1/true
    770   }
    771   compare g, 0x2b/plus
    772   {
    773     break-if-!=
    774     return 1/true
    775   }
    776   compare g, 0x2d/dash  # '-' not allowed in symbols
    777   {
    778     break-if-!=
    779     return 1/true
    780   }
    781   compare g, 0x2e/period
    782   {
    783     break-if-!=
    784     return 1/true
    785   }
    786   compare g, 0x2f/slash
    787   {
    788     break-if-!=
    789     return 1/true
    790   }
    791   compare g, 0x3a/colon
    792   {
    793     break-if-!=
    794     return 1/true
    795   }
    796   compare g, 0x3b/semi-colon
    797   {
    798     break-if-!=
    799     return 1/true
    800   }
    801   compare g, 0x3c/less-than
    802   {
    803     break-if-!=
    804     return 1/true
    805   }
    806   compare g, 0x3d/equal
    807   {
    808     break-if-!=
    809     return 1/true
    810   }
    811   compare g, 0x3e/greater-than
    812   {
    813     break-if-!=
    814     return 1/true
    815   }
    816   # '?' is a symbol char
    817   compare g, 0x5c/backslash
    818   {
    819     break-if-!=
    820     return 1/true
    821   }
    822   compare g, 0x5e/caret
    823   {
    824     break-if-!=
    825     return 1/true
    826   }
    827   # '_' is a symbol char
    828   compare g, 0x7c/vertical-line
    829   {
    830     break-if-!=
    831     return 1/true
    832   }
    833   compare g, 0x7e/tilde
    834   {
    835     break-if-!=
    836     return 1/true
    837   }
    838   return 0/false
    839 }
    840
    841 fn number-token? _in: (addr cell) -> _/eax: boolean {
    842   var in/eax: (addr cell) <- copy _in
    843   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    844   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    845   rewind-stream in-data
    846   var g/eax: grapheme <- read-grapheme in-data
    847   var result/eax: boolean <- decimal-digit? g
    848   return result
    849 }
    850
    851 fn bracket-token? _in: (addr cell) -> _/eax: boolean {
    852   var in/eax: (addr cell) <- copy _in
    853   {
    854     var in-type/eax: (addr int) <- get in, type
    855     compare *in-type, 3/stream
    856     break-if-!=
    857     # streams are never paren tokens
    858     return 0/false
    859   }
    860   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    861   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    862   rewind-stream in-data
    863   var g/eax: grapheme <- read-grapheme in-data
    864   var result/eax: boolean <- bracket-grapheme? g
    865   return result
    866 }
    867
    868 fn quote-token? _in: (addr cell) -> _/eax: boolean {
    869   var in/eax: (addr cell) <- copy _in
    870   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    871   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    872   rewind-stream in-data
    873   var result/eax: boolean <- stream-data-equal? in-data, "'"
    874   return result
    875 }
    876
    877 fn backquote-token? _in: (addr cell) -> _/eax: boolean {
    878   var in/eax: (addr cell) <- copy _in
    879   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    880   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    881   rewind-stream in-data
    882   var result/eax: boolean <- stream-data-equal? in-data, "`"
    883   return result
    884 }
    885
    886 fn unquote-token? _in: (addr cell) -> _/eax: boolean {
    887   var in/eax: (addr cell) <- copy _in
    888   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    889   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    890   rewind-stream in-data
    891   var result/eax: boolean <- stream-data-equal? in-data, ","
    892   return result
    893 }
    894
    895 fn unquote-splice-token? _in: (addr cell) -> _/eax: boolean {
    896   var in/eax: (addr cell) <- copy _in
    897   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    898   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    899   rewind-stream in-data
    900   var result/eax: boolean <- stream-data-equal? in-data, ",@"
    901   return result
    902 }
    903
    904 fn open-paren-token? _in: (addr cell) -> _/eax: boolean {
    905   var in/eax: (addr cell) <- copy _in
    906   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    907   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
    908   var in-data/ecx: (addr stream byte) <- copy _in-data
    909   rewind-stream in-data
    910   var g/eax: grapheme <- read-grapheme in-data
    911   compare g, 0x28/open-paren
    912   {
    913     break-if-!=
    914     var result/eax: boolean <- stream-empty? in-data
    915     return result
    916   }
    917   return 0/false
    918 }
    919
    920 fn close-paren-token? _in: (addr cell) -> _/eax: boolean {
    921   var in/eax: (addr cell) <- copy _in
    922   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    923   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
    924   var in-data/ecx: (addr stream byte) <- copy _in-data
    925   rewind-stream in-data
    926   var g/eax: grapheme <- read-grapheme in-data
    927   compare g, 0x29/close-paren
    928   {
    929     break-if-!=
    930     var result/eax: boolean <- stream-empty? in-data
    931     return result
    932   }
    933   return 0/false
    934 }
    935
    936 fn dot-token? _in: (addr cell) -> _/eax: boolean {
    937   var in/eax: (addr cell) <- copy _in
    938   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    939   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
    940   var in-data/ecx: (addr stream byte) <- copy _in-data
    941   rewind-stream in-data
    942   var g/eax: grapheme <- read-grapheme in-data
    943   compare g, 0x2e/dot
    944   {
    945     break-if-!=
    946     var result/eax: boolean <- stream-empty? in-data
    947     return result
    948   }
    949   return 0/false
    950 }
    951
    952 fn test-dot-token {
    953   var tmp-storage: (handle cell)
    954   var tmp-ah/eax: (addr handle cell) <- address tmp-storage
    955   new-symbol tmp-ah, "."
    956   var tmp/eax: (addr cell) <- lookup *tmp-ah
    957   var result/eax: boolean <- dot-token? tmp
    958   check result, "F - test-dot-token"
    959 }
    960
    961 fn stream-token? _in: (addr cell) -> _/eax: boolean {
    962   var in/eax: (addr cell) <- copy _in
    963   var in-type/eax: (addr int) <- get in, type
    964   compare *in-type, 3/stream
    965   {
    966     break-if-=
    967     return 0/false
    968   }
    969   return 1/true
    970 }
    971
    972 fn comment-token? _in: (addr cell) -> _/eax: boolean {
    973   var in/eax: (addr cell) <- copy _in
    974   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    975   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    976   rewind-stream in-data
    977   var g/eax: grapheme <- read-grapheme in-data
    978   compare g, 0x23/hash
    979   {
    980     break-if-=
    981     return 0/false
    982   }
    983   return 1/true
    984 }