https://github.com/akkartik/mu/blob/main/shell/tokenize.mu

      1 # We reuse the cell data structure for tokenization
      2 # Token cells are special, though. They have no type, they're always atoms,
      3 # they always have text-data.
      4
      5 fn tokenize in: (addr gap-buffer), out: (addr stream cell), trace: (addr trace) {
      6   trace-text trace, "read", "tokenize"
      7   trace-lower trace
      8   rewind-gap-buffer in
      9   var token-storage: cell
     10   var token/edx: (addr cell) <- address token-storage
     11   {
     12     skip-whitespace-from-gap-buffer in
     13     var done?/eax: boolean <- gap-buffer-scan-done? in
     14     compare done?, 0/false
     15     break-if-!=
     16     # initialize token data each iteration to avoid aliasing
     17     var dest-ah/eax: (addr handle stream byte) <- get token, text-data
     18     populate-stream dest-ah, 0x40/max-token-size
     19     #
     20     next-token in, token, trace
     21     var error?/eax: boolean <- has-errors? trace
     22     compare error?, 0/false
     23     {
     24       break-if-=
     25       return
     26     }
     27     write-to-stream out, token  # shallow-copy text-data
     28     loop
     29   }
     30   trace-higher trace
     31 }
     32
     33 fn next-token in: (addr gap-buffer), _out-cell: (addr cell), trace: (addr trace) {
     34   trace-text trace, "read", "next-token"
     35   trace-lower trace
     36   var out-cell/eax: (addr cell) <- copy _out-cell
     37   var out-ah/eax: (addr handle stream byte) <- get out-cell, text-data
     38   var _out/eax: (addr stream byte) <- lookup *out-ah
     39   var out/edi: (addr stream byte) <- copy _out
     40   $next-token:body: {
     41     clear-stream out
     42     skip-whitespace-from-gap-buffer in
     43     var g/eax: grapheme <- peek-from-gap-buffer in
     44     {
     45       var stream-storage: (stream byte 0x40)
     46       var stream/esi: (addr stream byte) <- address stream-storage
     47       write stream, "next: "
     48       var gval/eax: int <- copy g
     49       write-int32-hex stream, gval
     50       trace trace, "read", stream
     51     }
     52     # digit
     53     {
     54       var digit?/eax: boolean <- decimal-digit? g
     55       compare digit?, 0/false
     56       break-if-=
     57       next-number-token in, out, trace
     58       break $next-token:body
     59     }
     60     # other symbol char
     61     {
     62       var symbol?/eax: boolean <- symbol-grapheme? g
     63       compare symbol?, 0/false
     64       break-if-=
     65       next-symbol-token in, out, trace
     66       break $next-token:body
     67     }
     68     # brackets are always single-char tokens
     69     {
     70       var bracket?/eax: boolean <- bracket-grapheme? g
     71       compare bracket?, 0/false
     72       break-if-=
     73       var g/eax: grapheme <- read-from-gap-buffer in
     74       next-bracket-token g, out, trace
     75       break $next-token:body
     76     }
     77     # non-symbol operators
     78     {
     79       var operator?/eax: boolean <- operator-grapheme? g
     80       compare operator?, 0/false
     81       break-if-=
     82       next-operator-token in, out, trace
     83       break $next-token:body
     84     }
     85   }
     86   trace-higher trace
     87   var stream-storage: (stream byte 0x40)
     88   var stream/eax: (addr stream byte) <- address stream-storage
     89   write stream, "=> "
     90   rewind-stream out
     91   write-stream stream, out
     92   trace trace, "read", stream
     93 }
     94
     95 fn next-symbol-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
     96   trace-text trace, "read", "looking for a symbol"
     97   trace-lower trace
     98   $next-symbol-token:loop: {
     99     var done?/eax: boolean <- gap-buffer-scan-done? in
    100     compare done?, 0/false
    101     break-if-!=
    102     var g/eax: grapheme <- peek-from-gap-buffer in
    103     {
    104       var stream-storage: (stream byte 0x40)
    105       var stream/esi: (addr stream byte) <- address stream-storage
    106       write stream, "next: "
    107       var gval/eax: int <- copy g
    108       write-int32-hex stream, gval
    109       trace trace, "read", stream
    110     }
    111     # if non-symbol, return
    112     {
    113       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
    114       compare symbol-grapheme?, 0/false
    115       break-if-!=
    116       trace-text trace, "read", "stop"
    117       break $next-symbol-token:loop
    118     }
    119     var g/eax: grapheme <- read-from-gap-buffer in
    120     write-grapheme out, g
    121     loop
    122   }
    123   trace-higher trace
    124   var stream-storage: (stream byte 0x40)
    125   var stream/esi: (addr stream byte) <- address stream-storage
    126   write stream, "=> "
    127   rewind-stream out
    128   write-stream stream, out
    129   trace trace, "read", stream
    130 }
    131
    132 fn next-operator-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    133   trace-text trace, "read", "looking for a operator"
    134   trace-lower trace
    135   $next-operator-token:loop: {
    136     var done?/eax: boolean <- gap-buffer-scan-done? in
    137     compare done?, 0/false
    138     break-if-!=
    139     var g/eax: grapheme <- peek-from-gap-buffer in
    140     {
    141       var stream-storage: (stream byte 0x40)
    142       var stream/esi: (addr stream byte) <- address stream-storage
    143       write stream, "next: "
    144       var gval/eax: int <- copy g
    145       write-int32-hex stream, gval
    146       trace trace, "read", stream
    147     }
    148     # if non-operator, return
    149     {
    150       var operator-grapheme?/eax: boolean <- operator-grapheme? g
    151       compare operator-grapheme?, 0/false
    152       break-if-!=
    153       trace-text trace, "read", "stop"
    154       break $next-operator-token:loop
    155     }
    156     var g/eax: grapheme <- read-from-gap-buffer in
    157     write-grapheme out, g
    158     loop
    159   }
    160   trace-higher trace
    161   var stream-storage: (stream byte 0x40)
    162   var stream/esi: (addr stream byte) <- address stream-storage
    163   write stream, "=> "
    164   rewind-stream out
    165   write-stream stream, out
    166   trace trace, "read", stream
    167 }
    168
    169 fn next-number-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
    170   trace-text trace, "read", "looking for a number"
    171   trace-lower trace
    172   $next-number-token:loop: {
    173     var done?/eax: boolean <- gap-buffer-scan-done? in
    174     compare done?, 0/false
    175     break-if-!=
    176     var g/eax: grapheme <- peek-from-gap-buffer in
    177     {
    178       var stream-storage: (stream byte 0x40)
    179       var stream/esi: (addr stream byte) <- address stream-storage
    180       write stream, "next: "
    181       var gval/eax: int <- copy g
    182       write-int32-hex stream, gval
    183       trace trace, "read", stream
    184     }
    185     # if not symbol grapheme, return
    186     {
    187       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
    188       compare symbol-grapheme?, 0/false
    189       break-if-!=
    190       trace-text trace, "read", "stop"
    191       break $next-number-token:loop
    192     }
    193     # if not digit grapheme, abort
    194     {
    195       var digit?/eax: boolean <- decimal-digit? g
    196       compare digit?, 0/false
    197       break-if-!=
    198       error trace, "invalid number"
    199       return
    200     }
    201     trace-text trace, "read", "append"
    202     var g/eax: grapheme <- read-from-gap-buffer in
    203     write-grapheme out, g
    204     loop
    205   }
    206   trace-higher trace
    207 }
    208
    209 fn next-bracket-token g: grapheme, out: (addr stream byte), trace: (addr trace) {
    210   trace-text trace, "read", "bracket"
    211   write-grapheme out, g
    212   var stream-storage: (stream byte 0x40)
    213   var stream/esi: (addr stream byte) <- address stream-storage
    214   write stream, "=> "
    215   rewind-stream out
    216   write-stream stream, out
    217   trace trace, "read", stream
    218 }
    219
    220 fn symbol-grapheme? g: grapheme -> _/eax: boolean {
    221   ## whitespace
    222   compare g, 9/tab
    223   {
    224     break-if-!=
    225     return 0/false
    226   }
    227   compare g, 0xa/newline
    228   {
    229     break-if-!=
    230     return 0/false
    231   }
    232   compare g, 0x20/space
    233   {
    234     break-if-!=
    235     return 0/false
    236   }
    237   ## quotes
    238   compare g, 0x22/double-quote
    239   {
    240     break-if-!=
    241     return 0/false
    242   }
    243   compare g, 0x60/backquote
    244   {
    245     break-if-!=
    246     return 0/false
    247   }
    248   ## brackets
    249   compare g, 0x28/open-paren
    250   {
    251     break-if-!=
    252     return 0/false
    253   }
    254   compare g, 0x29/close-paren
    255   {
    256     break-if-!=
    257     return 0/false
    258   }
    259   compare g, 0x5b/open-square-bracket
    260   {
    261     break-if-!=
    262     return 0/false
    263   }
    264   compare g, 0x5d/close-square-bracket
    265   {
    266     break-if-!=
    267     return 0/false
    268   }
    269   compare g, 0x7b/open-curly-bracket
    270   {
    271     break-if-!=
    272     return 0/false
    273   }
    274   compare g, 0x7d/close-curly-bracket
    275   {
    276     break-if-!=
    277     return 0/false
    278   }
    279   # - other punctuation
    280   # '!' is a symbol char
    281   compare g, 0x23/hash
    282   {
    283     break-if-!=
    284     return 0/false
    285   }
    286   # '$' is a symbol char
    287   compare g, 0x25/percent
    288   {
    289     break-if-!=
    290     return 0/false
    291   }
    292   compare g, 0x26/ampersand
    293   {
    294     break-if-!=
    295     return 0/false
    296   }
    297   compare g, 0x27/single-quote
    298   {
    299     break-if-!=
    300     return 0/false
    301   }
    302   compare g, 0x2a/asterisk
    303   {
    304     break-if-!=
    305     return 0/false
    306   }
    307   compare g, 0x2b/plus
    308   {
    309     break-if-!=
    310     return 0/false
    311   }
    312   compare g, 0x2c/comma
    313   {
    314     break-if-!=
    315     return 0/false
    316   }
    317   compare g, 0x2d/dash  # '-' not allowed in symbols
    318   {
    319     break-if-!=
    320     return 0/false
    321   }
    322   compare g, 0x2e/period
    323   {
    324     break-if-!=
    325     return 0/false
    326   }
    327   compare g, 0x2f/slash
    328   {
    329     break-if-!=
    330     return 0/false
    331   }
    332   compare g, 0x3a/colon
    333   {
    334     break-if-!=
    335     return 0/false
    336   }
    337   compare g, 0x3b/semi-colon
    338   {
    339     break-if-!=
    340     return 0/false
    341   }
    342   compare g, 0x3c/less-than
    343   {
    344     break-if-!=
    345     return 0/false
    346   }
    347   compare g, 0x3d/equal
    348   {
    349     break-if-!=
    350     return 0/false
    351   }
    352   compare g, 0x3e/greater-than
    353   {
    354     break-if-!=
    355     return 0/false
    356   }
    357   # '?' is a symbol char
    358   compare g, 0x40/at-sign
    359   {
    360     break-if-!=
    361     return 0/false
    362   }
    363   compare g, 0x5c/backslash
    364   {
    365     break-if-!=
    366     return 0/false
    367   }
    368   compare g, 0x5e/caret
    369   {
    370     break-if-!=
    371     return 0/false
    372   }
    373   # '_' is a symbol char
    374   compare g, 0x7c/vertical-line
    375   {
    376     break-if-!=
    377     return 0/false
    378   }
    379   compare g, 0x7e/tilde
    380   {
    381     break-if-!=
    382     return 0/false
    383   }
    384   return 1/true
    385 }
    386
    387 fn bracket-grapheme? g: grapheme -> _/eax: boolean {
    388   compare g, 0x28/open-paren
    389   {
    390     break-if-!=
    391     return 1/true
    392   }
    393   compare g, 0x29/close-paren
    394   {
    395     break-if-!=
    396     return 1/true
    397   }
    398   compare g, 0x5b/open-square-bracket
    399   {
    400     break-if-!=
    401     return 1/true
    402   }
    403   compare g, 0x5d/close-square-bracket
    404   {
    405     break-if-!=
    406     return 1/true
    407   }
    408   compare g, 0x7b/open-curly-bracket
    409   {
    410     break-if-!=
    411     return 1/true
    412   }
    413   compare g, 0x7d/close-curly-bracket
    414   {
    415     break-if-!=
    416     return 1/true
    417   }
    418   return 0/false
    419 }
    420
    421 fn operator-grapheme? g: grapheme -> _/eax: boolean {
    422   # '$' is a symbol char
    423   compare g, 0x25/percent
    424   {
    425     break-if-!=
    426     return 1/false
    427   }
    428   compare g, 0x26/ampersand
    429   {
    430     break-if-!=
    431     return 1/true
    432   }
    433   compare g, 0x27/single-quote
    434   {
    435     break-if-!=
    436     return 1/true
    437   }
    438   compare g, 0x2a/asterisk
    439   {
    440     break-if-!=
    441     return 1/true
    442   }
    443   compare g, 0x2b/plus
    444   {
    445     break-if-!=
    446     return 1/true
    447   }
    448   compare g, 0x2c/comma
    449   {
    450     break-if-!=
    451     return 1/true
    452   }
    453   compare g, 0x2d/dash  # '-' not allowed in symbols
    454   {
    455     break-if-!=
    456     return 1/true
    457   }
    458   compare g, 0x2e/period
    459   {
    460     break-if-!=
    461     return 1/true
    462   }
    463   compare g, 0x2f/slash
    464   {
    465     break-if-!=
    466     return 1/true
    467   }
    468   compare g, 0x3a/colon
    469   {
    470     break-if-!=
    471     return 1/true
    472   }
    473   compare g, 0x3b/semi-colon
    474   {
    475     break-if-!=
    476     return 1/true
    477   }
    478   compare g, 0x3c/less-than
    479   {
    480     break-if-!=
    481     return 1/true
    482   }
    483   compare g, 0x3d/equal
    484   {
    485     break-if-!=
    486     return 1/true
    487   }
    488   compare g, 0x3e/greater-than
    489   {
    490     break-if-!=
    491     return 1/true
    492   }
    493   # '?' is a symbol char
    494   compare g, 0x40/at-sign
    495   {
    496     break-if-!=
    497     return 1/true
    498   }
    499   compare g, 0x5c/backslash
    500   {
    501     break-if-!=
    502     return 1/true
    503   }
    504   compare g, 0x5e/caret
    505   {
    506     break-if-!=
    507     return 1/true
    508   }
    509   # '_' is a symbol char
    510   compare g, 0x7c/vertical-line
    511   {
    512     break-if-!=
    513     return 1/true
    514   }
    515   compare g, 0x7e/tilde
    516   {
    517     break-if-!=
    518     return 1/true
    519   }
    520   return 0/false
    521 }
    522
    523 fn number-token? _in: (addr cell) -> _/eax: boolean {
    524   var in/eax: (addr cell) <- copy _in
    525   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    526   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    527   rewind-stream in-data
    528   var g/eax: grapheme <- read-grapheme in-data
    529   var result/eax: boolean <- decimal-digit? g
    530   return result
    531 }
    532
    533 fn bracket-token? _in: (addr cell) -> _/eax: boolean {
    534   var in/eax: (addr cell) <- copy _in
    535   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    536   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    537   rewind-stream in-data
    538   var g/eax: grapheme <- read-grapheme in-data
    539   var result/eax: boolean <- bracket-grapheme? g
    540   return result
    541 }
    542
    543 fn open-paren-token? _in: (addr cell) -> _/eax: boolean {
    544   var in/eax: (addr cell) <- copy _in
    545   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    546   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    547   rewind-stream in-data
    548   var g/eax: grapheme <- read-grapheme in-data
    549   compare g, 0x28/open-paren
    550   {
    551     break-if-!=
    552     return 1/true
    553   }
    554   return 0/false
    555 }
    556
    557 fn close-paren-token? _in: (addr cell) -> _/eax: boolean {
    558   var in/eax: (addr cell) <- copy _in
    559   var in-data-ah/eax: (addr handle stream byte) <- get in, text-data
    560   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
    561   rewind-stream in-data
    562   var g/eax: grapheme <- read-grapheme in-data
    563   compare g, 0x29/open-paren
    564   {
    565     break-if-!=
    566     return 1/true
    567   }
    568   return 0/false
    569 }