https://github.com/akkartik/mu/blob/main/shell/tokenize.mu

       1 # The language is indent-sensitive.
       2 # Each line consists of an initial indent token followed by other tokens.
       3 type token {
       4   type: int
       5   # type 0: default
       6   # type 1: stream
       7   text-data: (handle stream byte)
       8   # type 2: skip (end of line or end of file)
       9   # type 3: indent
      10   number-data: int
      11 }
      12
      13 fn tokenize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
      14   trace-text trace, "tokenize", "tokenize"
      15   trace-lower trace
      16   rewind-gap-buffer in
      17   var at-start-of-line?/edi: boolean <- copy 1/true
      18   {
      19     var done?/eax: boolean <- gap-buffer-scan-done? in
      20     compare done?, 0/false
      21     break-if-!=
      22     #
      23     var token-storage: token
      24     var token/edx: (addr token) <- address token-storage
      25     at-start-of-line? <- next-token in, token, at-start-of-line?, trace
      26     var error?/eax: boolean <- has-errors? trace
      27     compare error?, 0/false
      28     {
      29       break-if-=
      30       return
      31     }
      32     var skip?/eax: boolean <- skip-token? token
      33     compare skip?, 0/false
      34     loop-if-!=
      35     write-to-stream out, token  # shallow-copy text-data
      36     loop
      37   }
      38   trace-higher trace
      39 }
      40
      41 fn test-tokenize-number {
      42   var in-storage: gap-buffer
      43   var in/esi: (addr gap-buffer) <- address in-storage
      44   initialize-gap-buffer-with in, "123 a"
      45   #
      46   var stream-storage: (stream token 0x10)
      47   var stream/edi: (addr stream token) <- address stream-storage
      48   #
      49   var trace-storage: trace
      50   var trace/edx: (addr trace) <- address trace-storage
      51   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
      52   tokenize in, stream, trace
      53   #
      54   var curr-token-storage: token
      55   var curr-token/ebx: (addr token) <- address curr-token-storage
      56   read-from-stream stream, curr-token
      57   var curr-token-type/eax: (addr int) <- get curr-token, type
      58   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-number/before-indent-type"
      59   var curr-token-data/eax: (addr int) <- get curr-token, number-data
      60   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-number/before-indent"
      61   read-from-stream stream, curr-token
      62   var number?/eax: boolean <- number-token? curr-token
      63   check number?, "F - test-tokenize-number"
      64   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
      65   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
      66   check-stream-equal curr-token-data, "123", "F - test-tokenize-number: value"
      67 }
      68
      69 fn test-tokenize-negative-number {
      70   var in-storage: gap-buffer
      71   var in/esi: (addr gap-buffer) <- address in-storage
      72   initialize-gap-buffer-with in, "-123 a"
      73   #
      74   var stream-storage: (stream token 0x10)
      75   var stream/edi: (addr stream token) <- address stream-storage
      76   #
      77   var trace-storage: trace
      78   var trace/edx: (addr trace) <- address trace-storage
      79   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
      80   tokenize in, stream, trace
      81   #
      82   var curr-token-storage: token
      83   var curr-token/ebx: (addr token) <- address curr-token-storage
      84   read-from-stream stream, curr-token
      85   var curr-token-type/eax: (addr int) <- get curr-token, type
      86   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-negative-number/before-indent-type"
      87   var curr-token-data/eax: (addr int) <- get curr-token, number-data
      88   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-negative-number/before-indent"
      89   read-from-stream stream, curr-token
      90   var number?/eax: boolean <- number-token? curr-token
      91   check number?, "F - test-tokenize-negative-number"
      92   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
      93   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
      94   check-stream-equal curr-token-data, "-123", "F - test-tokenize-negative-number: value"
      95 }
      96
      97 fn test-tokenize-quote {
      98   var in-storage: gap-buffer
      99   var in/esi: (addr gap-buffer) <- address in-storage
     100   initialize-gap-buffer-with in, "'(a)"
     101   #
     102   var stream-storage: (stream token 0x10)
     103   var stream/edi: (addr stream token) <- address stream-storage
     104   #
     105   var trace-storage: trace
     106   var trace/edx: (addr trace) <- address trace-storage
     107   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     108   tokenize in, stream, trace
     109   #
     110   var curr-token-storage: token
     111   var curr-token/ebx: (addr token) <- address curr-token-storage
     112   read-from-stream stream, curr-token
     113   var curr-token-type/eax: (addr int) <- get curr-token, type
     114   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-quote/before-indent-type"
     115   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     116   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-quote/before-indent"
     117   read-from-stream stream, curr-token
     118   var quote?/eax: boolean <- quote-token? curr-token
     119   check quote?, "F - test-tokenize-quote: quote"
     120   read-from-stream stream, curr-token
     121   var open-paren?/eax: boolean <- open-paren-token? curr-token
     122   check open-paren?, "F - test-tokenize-quote: open paren"
     123   read-from-stream stream, curr-token  # skip a
     124   read-from-stream stream, curr-token
     125   var close-paren?/eax: boolean <- close-paren-token? curr-token
     126   check close-paren?, "F - test-tokenize-quote: close paren"
     127 }
     128
     129 fn test-tokenize-backquote {
     130   var in-storage: gap-buffer
     131   var in/esi: (addr gap-buffer) <- address in-storage
     132   initialize-gap-buffer-with in, "`(a)"
     133   #
     134   var stream-storage: (stream token 0x10)
     135   var stream/edi: (addr stream token) <- address stream-storage
     136   #
     137   var trace-storage: trace
     138   var trace/edx: (addr trace) <- address trace-storage
     139   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     140   tokenize in, stream, trace
     141   #
     142   var curr-token-storage: token
     143   var curr-token/ebx: (addr token) <- address curr-token-storage
     144   read-from-stream stream, curr-token
     145   var curr-token-type/eax: (addr int) <- get curr-token, type
     146   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-backquote/before-indent-type"
     147   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     148   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-backquote/before-indent"
     149   read-from-stream stream, curr-token
     150   var backquote?/eax: boolean <- backquote-token? curr-token
     151   check backquote?, "F - test-tokenize-backquote: backquote"
     152   read-from-stream stream, curr-token
     153   var open-paren?/eax: boolean <- open-paren-token? curr-token
     154   check open-paren?, "F - test-tokenize-backquote: open paren"
     155   read-from-stream stream, curr-token  # skip a
     156   read-from-stream stream, curr-token
     157   var close-paren?/eax: boolean <- close-paren-token? curr-token
     158   check close-paren?, "F - test-tokenize-backquote: close paren"
     159 }
     160
     161 fn test-tokenize-unquote {
     162   var in-storage: gap-buffer
     163   var in/esi: (addr gap-buffer) <- address in-storage
     164   initialize-gap-buffer-with in, ",(a)"
     165   #
     166   var stream-storage: (stream token 0x10)
     167   var stream/edi: (addr stream token) <- address stream-storage
     168   #
     169   var trace-storage: trace
     170   var trace/edx: (addr trace) <- address trace-storage
     171   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     172   tokenize in, stream, trace
     173   #
     174   var curr-token-storage: token
     175   var curr-token/ebx: (addr token) <- address curr-token-storage
     176   read-from-stream stream, curr-token
     177   var curr-token-type/eax: (addr int) <- get curr-token, type
     178   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-unquote/before-indent-type"
     179   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     180   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-unquote/before-indent"
     181   read-from-stream stream, curr-token
     182   var unquote?/eax: boolean <- unquote-token? curr-token
     183   check unquote?, "F - test-tokenize-unquote: unquote"
     184   read-from-stream stream, curr-token
     185   var open-paren?/eax: boolean <- open-paren-token? curr-token
     186   check open-paren?, "F - test-tokenize-unquote: open paren"
     187   read-from-stream stream, curr-token  # skip a
     188   read-from-stream stream, curr-token
     189   var close-paren?/eax: boolean <- close-paren-token? curr-token
     190   check close-paren?, "F - test-tokenize-unquote: close paren"
     191 }
     192
     193 fn test-tokenize-unquote-splice {
     194   var in-storage: gap-buffer
     195   var in/esi: (addr gap-buffer) <- address in-storage
     196   initialize-gap-buffer-with in, ",@a"
     197   #
     198   var stream-storage: (stream token 0x10)
     199   var stream/edi: (addr stream token) <- address stream-storage
     200   #
     201   var trace-storage: trace
     202   var trace/edx: (addr trace) <- address trace-storage
     203   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     204   tokenize in, stream, trace
     205   #
     206   var curr-token-storage: token
     207   var curr-token/ebx: (addr token) <- address curr-token-storage
     208   read-from-stream stream, curr-token
     209   var curr-token-type/eax: (addr int) <- get curr-token, type
     210   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-unquote-splice/before-indent-type"
     211   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     212   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-unquote-splice/before-indent"
     213   read-from-stream stream, curr-token
     214   var unquote-splice?/eax: boolean <- unquote-splice-token? curr-token
     215   check unquote-splice?, "F - test-tokenize-unquote-splice: unquote-splice"
     216 }
     217
     218 fn test-tokenize-dotted-list {
     219   var in-storage: gap-buffer
     220   var in/esi: (addr gap-buffer) <- address in-storage
     221   initialize-gap-buffer-with in, "(a . b)"
     222   #
     223   var stream-storage: (stream token 0x10)
     224   var stream/edi: (addr stream token) <- address stream-storage
     225   #
     226   var trace-storage: trace
     227   var trace/edx: (addr trace) <- address trace-storage
     228   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     229   tokenize in, stream, trace
     230   #
     231   var curr-token-storage: token
     232   var curr-token/ebx: (addr token) <- address curr-token-storage
     233   read-from-stream stream, curr-token
     234   var curr-token-type/eax: (addr int) <- get curr-token, type
     235   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-dotted-list/before-indent-type"
     236   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     237   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-dotted-list/before-indent"
     238   read-from-stream stream, curr-token
     239   var open-paren?/eax: boolean <- open-paren-token? curr-token
     240   check open-paren?, "F - test-tokenize-dotted-list: open paren"
     241   read-from-stream stream, curr-token  # skip a
     242   read-from-stream stream, curr-token
     243   var dot?/eax: boolean <- dot-token? curr-token
     244   check dot?, "F - test-tokenize-dotted-list: dot"
     245   read-from-stream stream, curr-token  # skip b
     246   read-from-stream stream, curr-token
     247   var close-paren?/eax: boolean <- close-paren-token? curr-token
     248   check close-paren?, "F - test-tokenize-dotted-list: close paren"
     249 }
     250
     251 fn test-tokenize-stream-literal {
     252   var in-storage: gap-buffer
     253   var in/esi: (addr gap-buffer) <- address in-storage
     254   initialize-gap-buffer-with in, "[abc def]"
     255   #
     256   var stream-storage: (stream token 0x10)
     257   var stream/edi: (addr stream token) <- address stream-storage
     258   #
     259   var trace-storage: trace
     260   var trace/edx: (addr trace) <- address trace-storage
     261   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     262   tokenize in, stream, trace
     263   #
     264   var curr-token-storage: token
     265   var curr-token/ebx: (addr token) <- address curr-token-storage
     266   read-from-stream stream, curr-token
     267   var curr-token-type/eax: (addr int) <- get curr-token, type
     268   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-stream-literal/before-indent-type"
     269   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     270   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-stream-literal/before-indent"
     271   read-from-stream stream, curr-token
     272   var stream?/eax: boolean <- stream-token? curr-token
     273   check stream?, "F - test-tokenize-stream-literal: type"
     274   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
     275   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
     276   var data-equal?/eax: boolean <- stream-data-equal? curr-token-data, "abc def"
     277   check data-equal?, "F - test-tokenize-stream-literal"
     278   var empty?/eax: boolean <- stream-empty? stream
     279   check empty?, "F - test-tokenize-stream-literal: empty?"
     280 }
     281
     282 fn test-tokenize-stream-literal-in-tree {
     283   var in-storage: gap-buffer
     284   var in/esi: (addr gap-buffer) <- address in-storage
     285   initialize-gap-buffer-with in, "([abc def])"
     286   #
     287   var stream-storage: (stream token 0x10)
     288   var stream/edi: (addr stream token) <- address stream-storage
     289   #
     290   var trace-storage: trace
     291   var trace/edx: (addr trace) <- address trace-storage
     292   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     293   tokenize in, stream, trace
     294   #
     295   var curr-token-storage: token
     296   var curr-token/ebx: (addr token) <- address curr-token-storage
     297   read-from-stream stream, curr-token
     298   var curr-token-type/eax: (addr int) <- get curr-token, type
     299   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-stream-literal-in-tree/before-indent-type"
     300   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     301   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-stream-literal-in-tree/before-indent"
     302   read-from-stream stream, curr-token
     303   var bracket?/eax: boolean <- bracket-token? curr-token
     304   check bracket?, "F - test-tokenize-stream-literal-in-tree: open paren"
     305   read-from-stream stream, curr-token
     306   var stream?/eax: boolean <- stream-token? curr-token
     307   check stream?, "F - test-tokenize-stream-literal-in-tree: type"
     308   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
     309   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
     310   var data-equal?/eax: boolean <- stream-data-equal? curr-token-data, "abc def"
     311   check data-equal?, "F - test-tokenize-stream-literal-in-tree"
     312   read-from-stream stream, curr-token
     313   var bracket?/eax: boolean <- bracket-token? curr-token
     314   check bracket?, "F - test-tokenize-stream-literal-in-tree: close paren"
     315   var empty?/eax: boolean <- stream-empty? stream
     316   check empty?, "F - test-tokenize-stream-literal-in-tree: empty?"
     317 }
     318
     319 fn test-tokenize-indent {
     320   var in-storage: gap-buffer
     321   var in/esi: (addr gap-buffer) <- address in-storage
     322   initialize-gap-buffer-with in, "abc\n  def"
     323   #
     324   var stream-storage: (stream token 0x10)
     325   var stream/edi: (addr stream token) <- address stream-storage
     326   #
     327   var trace-storage: trace
     328   var trace/edx: (addr trace) <- address trace-storage
     329   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
     330   tokenize in, stream, trace
     331   #
     332   var curr-token-storage: token
     333   var curr-token/ebx: (addr token) <- address curr-token-storage
     334   read-from-stream stream, curr-token
     335   var curr-token-type/eax: (addr int) <- get curr-token, type
     336   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-indent/before-indent-type"
     337   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     338   check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-indent/before-indent"
     339   read-from-stream stream, curr-token
     340   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
     341   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
     342   check-stream-equal curr-token-data, "abc", "F - test-tokenize-indent/before"
     343   #
     344   read-from-stream stream, curr-token
     345   var curr-token-type/eax: (addr int) <- get curr-token, type
     346   check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-indent/type"
     347   var curr-token-data/eax: (addr int) <- get curr-token, number-data
     348   check-ints-equal *curr-token-data, 2/spaces, "F - test-tokenize-indent"
     349   #
     350   read-from-stream stream, curr-token
     351   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
     352   var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
     353   check-stream-equal curr-token-data, "def", "F - test-tokenize-indent/after"
     354 }
     355
     356 # caller is responsible for threading start-of-line? between calls to next-token
     357 # 'in' may contain whitespace if start-of-line?
     358 fn next-token in: (addr gap-buffer), out: (addr token), start-of-line?: boolean, trace: (addr trace) -> _/edi: boolean {
     359   trace-text trace, "tokenize", "next-token"
     360   trace-lower trace
     361   # save an indent token if necessary
     362   {
     363     compare start-of-line?, 0/false
     364     break-if-=
     365     next-indent-token in, out, trace  # might not be returned
     366   }
     367   skip-spaces-from-gap-buffer in
     368   var g/eax: grapheme <- peek-from-gap-buffer in
     369   {
     370     compare g, 0x23/comment
     371     break-if-!=
     372     skip-rest-of-line in
     373   }
     374   var g/eax: grapheme <- peek-from-gap-buffer in
     375   {
     376     compare g, 0xa/newline
     377     break-if-!=
     378     trace-text trace, "tokenize", "newline"
     379     g <- read-from-gap-buffer in
     380     initialize-skip-token out  # might drop indent if that's all there was in this line
     381     trace-higher trace
     382     return 1/at-start-of-line
     383   }
     384   {
     385     compare start-of-line?, 0/false
     386     break-if-=
     387     # still here? no comment or newline? return saved indent
     388     trace-higher trace
     389     return 0/not-at-start-of-line
     390   }
     391   {
     392     var done?/eax: boolean <- gap-buffer-scan-done? in
     393     compare done?, 0/false
     394     break-if-=
     395     trace-text trace, "tokenize", "end"
     396     initialize-skip-token out
     397     trace-higher trace
     398     return 1/at-start-of-line
     399   }
     400   var _g/eax: grapheme <- peek-from-gap-buffer in
     401   var g/ecx: grapheme <- copy _g
     402   {
     403     var should-trace?/eax: boolean <- should-trace? trace
     404     compare should-trace?, 0/false
     405     break-if-=
     406     var stream-storage: (stream byte 0x40)
     407     var stream/esi: (addr stream byte) <- address stream-storage
     408     write stream, "next: "
     409     var gval/eax: int <- copy g
     410     write-int32-hex stream, gval
     411     trace trace, "tokenize", stream
     412   }
     413   $next-token:case: {
     414     # open square brackets begin streams
     415     {
     416       compare g, 0x5b/open-square-bracket
     417       break-if-!=
     418       var dummy/eax: grapheme <- read-from-gap-buffer in  # skip open bracket
     419       next-stream-token in, out, trace
     420       break $next-token:case
     421     }
     422     # other symbol char
     423     {
     424       var symbol?/eax: boolean <- symbol-grapheme? g
     425       compare symbol?, 0/false
     426       break-if-=
     427       next-symbol-token in, out, trace
     428       break $next-token:case
     429     }
     430     # unbalanced close square brackets are errors
     431     {
     432       compare g, 0x5d/close-square-bracket
     433       break-if-!=
     434       error trace, "unbalanced ']'"
     435       return start-of-line?
     436     }
     437     # other brackets are always single-char tokens
     438     {
     439       var bracket?/eax: boolean <- bracket-grapheme? g
     440       compare bracket?, 0/false
     441       break-if-=
     442       var g/eax: grapheme <- read-from-gap-buffer in
     443       next-bracket-token g, out, trace
     444       break $next-token:case
     445     }
     446     # quote
     447     {
     448       compare g, 0x27/single-quote
     449       break-if-!=
     450       var g/eax: grapheme <- read-from-gap-buffer in  # consume
     451       initialize-token out, "'"
     452       break $next-token:case
     453     }
     454     # backquote
     455     {
     456       compare g, 0x60/backquote
     457       break-if-!=
     458       var g/eax: grapheme <- read-from-gap-buffer in  # consume
     459       initialize-token out, "`"
     460       break $next-token:case
     461     }
     462     # unquote
     463     {
     464       compare g, 0x2c/comma
     465       break-if-!=
     466       var g/eax: grapheme <- read-from-gap-buffer in  # consume
     467       # check for unquote-splice
     468       {
     469         g <- peek-from-gap-buffer in
     470         compare g, 0x40/at-sign
     471         break-if-!=
     472         g <- read-from-gap-buffer in
     473         initialize-token out, ",@"
     474         break $next-token:case
     475       }
     476       initialize-token out, ","
     477       break $next-token:case
     478     }
     479     set-cursor-position 0/screen, 0x40 0x20
     480     {
     481       var foo/eax: int <- copy g
     482       draw-int32-decimal-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, foo, 7/fg 0/bg
     483     }
     484     abort "unknown token type"
     485   }
     486   trace-higher trace
     487   {
     488     var should-trace?/eax: boolean <- should-trace? trace
     489     compare should-trace?, 0/false
     490     break-if-=
     491     var stream-storage: (stream byte 0x400)  # maximum possible token size (next-stream-token)
     492     var stream/eax: (addr stream byte) <- address stream-storage
     493     write stream, "=> "
     494     write-token-text-data stream, out
     495     trace trace, "tokenize", stream
     496   }
     497   return start-of-line?
     498 }
     499
     500 fn next-symbol-token in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
     501   trace-text trace, "tokenize", "looking for a symbol"
     502   trace-lower trace
     503   var out/eax: (addr token) <- copy _out
     504   var out-data-ah/eax: (addr handle stream byte) <- get out, text-data
     505   populate-stream out-data-ah, 0x40/max-symbol-size
     506   var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
     507   var out-data/edi: (addr stream byte) <- copy _out-data
     508   $next-symbol-token:loop: {
     509     var done?/eax: boolean <- gap-buffer-scan-done? in
     510     compare done?, 0/false
     511     break-if-!=
     512     var g/eax: grapheme <- peek-from-gap-buffer in
     513     {
     514       {
     515         var should-trace?/eax: boolean <- should-trace? trace
     516         compare should-trace?, 0/false
     517       }
     518       break-if-=
     519       var stream-storage: (stream byte 0x40)
     520       var stream/esi: (addr stream byte) <- address stream-storage
     521       write stream, "next: "
     522       var gval/eax: int <- copy g
     523       write-int32-hex stream, gval
     524       trace trace, "tokenize", stream
     525     }
     526     # if non-symbol, return
     527     {
     528       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
     529       compare symbol-grapheme?, 0/false
     530       break-if-!=
     531       trace-text trace, "tokenize", "stop"
     532       break $next-symbol-token:loop
     533     }
     534     var g/eax: grapheme <- read-from-gap-buffer in
     535     write-grapheme out-data, g
     536     loop
     537   }
     538   trace-higher trace
     539   {
     540     var should-trace?/eax: boolean <- should-trace? trace
     541     compare should-trace?, 0/false
     542     break-if-=
     543     var stream-storage: (stream byte 0x40)
     544     var stream/esi: (addr stream byte) <- address stream-storage
     545     write stream, "=> "
     546     rewind-stream out-data
     547     write-stream stream, out-data
     548     trace trace, "tokenize", stream
     549   }
     550 }
     551
     552 fn next-number-token in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
     553   trace-text trace, "tokenize", "looking for a number"
     554   trace-lower trace
     555   var out/eax: (addr token) <- copy _out
     556   var out-data-ah/eax: (addr handle stream byte) <- get out, text-data
     557   populate-stream out-data-ah, 0x40
     558   var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
     559   var out-data/edi: (addr stream byte) <- copy _out-data
     560   $next-number-token:check-minus: {
     561     var g/eax: grapheme <- peek-from-gap-buffer in
     562     compare g, 0x2d/minus
     563     g <- read-from-gap-buffer in  # consume
     564     write-grapheme out-data, g
     565   }
     566   $next-number-token:loop: {
     567     var done?/eax: boolean <- gap-buffer-scan-done? in
     568     compare done?, 0/false
     569     break-if-!=
     570     var g/eax: grapheme <- peek-from-gap-buffer in
     571     {
     572       {
     573         var should-trace?/eax: boolean <- should-trace? trace
     574         compare should-trace?, 0/false
     575       }
     576       break-if-=
     577       var stream-storage: (stream byte 0x40)
     578       var stream/esi: (addr stream byte) <- address stream-storage
     579       write stream, "next: "
     580       var gval/eax: int <- copy g
     581       write-int32-hex stream, gval
     582       trace trace, "tokenize", stream
     583     }
     584     # if not symbol grapheme, return
     585     {
     586       var symbol-grapheme?/eax: boolean <- symbol-grapheme? g
     587       compare symbol-grapheme?, 0/false
     588       break-if-!=
     589       trace-text trace, "tokenize", "stop"
     590       break $next-number-token:loop
     591     }
     592     # if not digit grapheme, abort
     593     {
     594       var digit?/eax: boolean <- decimal-digit? g
     595       compare digit?, 0/false
     596       break-if-!=
     597       error trace, "invalid number"
     598       return
     599     }
     600     trace-text trace, "tokenize", "append"
     601     var g/eax: grapheme <- read-from-gap-buffer in
     602     write-grapheme out-data, g
     603     loop
     604   }
     605   trace-higher trace
     606 }
     607
     608 fn next-stream-token in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
     609   trace-text trace, "tokenize", "stream"
     610   var out/edi: (addr token) <- copy _out
     611   var out-type/eax: (addr int) <- get out, type
     612   copy-to *out-type, 1/stream
     613   var out-data-ah/eax: (addr handle stream byte) <- get out, text-data
     614   # stream tokens contain whole function definitions on boot, so we always
     615   # give them plenty of space
     616   populate-stream out-data-ah, 0x400/max-definition-size=1KB
     617   var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
     618   var out-data/edi: (addr stream byte) <- copy _out-data
     619   {
     620     var empty?/eax: boolean <- gap-buffer-scan-done? in
     621     compare empty?, 0/false
     622     {
     623       break-if-=
     624       error trace, "unbalanced '['"
     625       return
     626     }
     627     var g/eax: grapheme <- read-from-gap-buffer in
     628     compare g, 0x5d/close-square-bracket
     629     break-if-=
     630     write-grapheme out-data, g
     631     loop
     632   }
     633   {
     634     var should-trace?/eax: boolean <- should-trace? trace
     635     compare should-trace?, 0/false
     636     break-if-=
     637     var stream-storage: (stream byte 0x400)  # max-definition-size
     638     var stream/esi: (addr stream byte) <- address stream-storage
     639     write stream, "=> "
     640     rewind-stream out-data
     641     write-stream-immutable stream, out-data
     642     trace trace, "tokenize", stream
     643   }
     644 }
     645
     646 fn next-bracket-token g: grapheme, _out: (addr token), trace: (addr trace) {
     647   trace-text trace, "tokenize", "bracket"
     648   var out/eax: (addr token) <- copy _out
     649   var out-data-ah/eax: (addr handle stream byte) <- get out, text-data
     650   populate-stream out-data-ah, 0x40
     651   var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
     652   var out-data/edi: (addr stream byte) <- copy _out-data
     653   write-grapheme out-data, g
     654   {
     655     var should-trace?/eax: boolean <- should-trace? trace
     656     compare should-trace?, 0/false
     657     break-if-=
     658     var stream-storage: (stream byte 0x40)
     659     var stream/esi: (addr stream byte) <- address stream-storage
     660     write stream, "=> "
     661     rewind-stream out-data
     662     write-stream stream, out-data
     663     trace trace, "tokenize", stream
     664   }
     665 }
     666
     667 fn skip-rest-of-line in: (addr gap-buffer) {
     668   {
     669     var done?/eax: boolean <- gap-buffer-scan-done? in
     670     compare done?, 0/false
     671     break-if-!=
     672     var g/eax: grapheme <- peek-from-gap-buffer in
     673     compare g, 0xa/newline
     674     break-if-=
     675     g <- read-from-gap-buffer in  # consume
     676     loop
     677   }
     678 }
     679
     680 fn next-indent-token in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
     681   trace-text trace, "tokenize", "indent"
     682   trace-lower trace
     683   var out/edi: (addr token) <- copy _out
     684   var out-type/eax: (addr int) <- get out, type
     685   copy-to *out-type, 3/indent
     686   var dest/edi: (addr int) <- get out, number-data
     687   copy-to *dest, 0
     688   {
     689     var done?/eax: boolean <- gap-buffer-scan-done? in
     690     compare done?, 0/false
     691     break-if-!=
     692     var g/eax: grapheme <- peek-from-gap-buffer in
     693     {
     694       {
     695         var should-trace?/eax: boolean <- should-trace? trace
     696         compare should-trace?, 0/false
     697       }
     698       break-if-=
     699       var stream-storage: (stream byte 0x40)
     700       var stream/esi: (addr stream byte) <- address stream-storage
     701       write stream, "next: "
     702       var gval/eax: int <- copy g
     703       write-int32-hex stream, gval
     704       trace trace, "tokenize", stream
     705     }
     706     # if non-space, break
     707     compare g, 0x20/space
     708     break-if-!=
     709     g <- read-from-gap-buffer in
     710     increment *dest
     711     loop
     712   }
     713   trace-higher trace
     714   {
     715     var should-trace?/eax: boolean <- should-trace? trace
     716     compare should-trace?, 0/false
     717     break-if-=
     718     var stream-storage: (stream byte 0x40)
     719     var stream/esi: (addr stream byte) <- address stream-storage
     720     write stream, "=> indent "
     721     write-int32-hex stream, *dest
     722     trace trace, "tokenize", stream
     723   }
     724 }
     725
     726 # Mu carves up the space of graphemes into 4 categories:
     727 #   whitespace
     728 #   quotes and unquotes (from a Lisp perspective; doesn't include double
     729 #                        quotes or other Unicode quotes)
     730 #   operators
     731 #   symbols
     732 # (Numbers have their own parsing rules that don't fit cleanly in this
     733 # partition.)
     734 #
     735 # During tokenization operators and symbols are treated identically.
     736 # A later phase digs into that nuance.
     737
     738 fn symbol-grapheme? g: grapheme -> _/eax: boolean {
     739   var whitespace?/eax: boolean <- whitespace-grapheme? g
     740   compare whitespace?, 0/false
     741   {
     742     break-if-=
     743     return 0/false
     744   }
     745   var quote-or-unquote?/eax: boolean <- quote-or-unquote-grapheme? g
     746   compare quote-or-unquote?, 0/false
     747   {
     748     break-if-=
     749     return 0/false
     750   }
     751   var bracket?/eax: boolean <- bracket-grapheme? g
     752   compare bracket?, 0/false
     753   {
     754     break-if-=
     755     return 0/false
     756   }
     757   compare g, 0x23/hash  # comments get filtered out
     758   {
     759     break-if-!=
     760     return 0/false
     761   }
     762   compare g, 0x22/double-quote  # double quotes reserved for now
     763   {
     764     break-if-!=
     765     return 0/false
     766   }
     767   return 1/true
     768 }
     769
     770 fn whitespace-grapheme? g: grapheme -> _/eax: boolean {
     771   compare g, 9/tab
     772   {
     773     break-if-!=
     774     return 1/true
     775   }
     776   compare g, 0xa/newline
     777   {
     778     break-if-!=
     779     return 1/true
     780   }
     781   compare g, 0x20/space
     782   {
     783     break-if-!=
     784     return 1/true
     785   }
     786   return 0/false
     787 }
     788
     789 fn quote-or-unquote-grapheme? g: grapheme -> _/eax: boolean {
     790   compare g, 0x27/single-quote
     791   {
     792     break-if-!=
     793     return 1/true
     794   }
     795   compare g, 0x60/backquote
     796   {
     797     break-if-!=
     798     return 1/true
     799   }
     800   compare g, 0x2c/comma
     801   {
     802     break-if-!=
     803     return 1/true
     804   }
     805   compare g, 0x40/at-sign
     806   {
     807     break-if-!=
     808     return 1/true
     809   }
     810   return 0/false
     811 }
     812
     813 fn bracket-grapheme? g: grapheme -> _/eax: boolean {
     814   compare g, 0x28/open-paren
     815   {
     816     break-if-!=
     817     return 1/true
     818   }
     819   compare g, 0x29/close-paren
     820   {
     821     break-if-!=
     822     return 1/true
     823   }
     824   compare g, 0x5b/open-square-bracket
     825   {
     826     break-if-!=
     827     return 1/true
     828   }
     829   compare g, 0x5d/close-square-bracket
     830   {
     831     break-if-!=
     832     return 1/true
     833   }
     834   compare g, 0x7b/open-curly-bracket
     835   {
     836     break-if-!=
     837     return 1/true
     838   }
     839   compare g, 0x7d/close-curly-bracket
     840   {
     841     break-if-!=
     842     return 1/true
     843   }
     844   return 0/false
     845 }
     846
     847 fn number-token? _self: (addr token) -> _/eax: boolean {
     848   var self/eax: (addr token) <- copy _self
     849   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     850   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
     851   var in-data/ecx: (addr stream byte) <- copy _in-data
     852   rewind-stream in-data
     853   var g/eax: grapheme <- read-grapheme in-data
     854   # if '-', read another
     855   {
     856     compare g, 0x2d/minus
     857     break-if-!=
     858     g <- read-grapheme in-data
     859   }
     860   {
     861     {
     862       var result/eax: boolean <- decimal-digit? g
     863       compare result, 0/false
     864       break-if-!=
     865       return 0/false
     866     }
     867     {
     868       var done?/eax: boolean <- stream-empty? in-data
     869       compare done?, 0/false
     870     }
     871     break-if-!=
     872     g <- read-grapheme in-data
     873     loop
     874   }
     875   return 1/true
     876 }
     877
     878 fn bracket-token? _self: (addr token) -> _/eax: boolean {
     879   var self/eax: (addr token) <- copy _self
     880   {
     881     var in-type/eax: (addr int) <- get self, type
     882     compare *in-type, 1/stream
     883     break-if-!=
     884     # streams are never paren tokens
     885     return 0/false
     886   }
     887   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     888   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
     889   rewind-stream in-data
     890   var g/eax: grapheme <- read-grapheme in-data
     891   var result/eax: boolean <- bracket-grapheme? g
     892   return result
     893 }
     894
     895 fn quote-token? _self: (addr token) -> _/eax: boolean {
     896   var self/eax: (addr token) <- copy _self
     897   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     898   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
     899   rewind-stream in-data
     900   var result/eax: boolean <- stream-data-equal? in-data, "'"
     901   return result
     902 }
     903
     904 fn backquote-token? _self: (addr token) -> _/eax: boolean {
     905   var self/eax: (addr token) <- copy _self
     906   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     907   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
     908   rewind-stream in-data
     909   var result/eax: boolean <- stream-data-equal? in-data, "`"
     910   return result
     911 }
     912
     913 fn unquote-token? _self: (addr token) -> _/eax: boolean {
     914   var self/eax: (addr token) <- copy _self
     915   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     916   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
     917   rewind-stream in-data
     918   var result/eax: boolean <- stream-data-equal? in-data, ","
     919   return result
     920 }
     921
     922 fn unquote-splice-token? _self: (addr token) -> _/eax: boolean {
     923   var self/eax: (addr token) <- copy _self
     924   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     925   var in-data/eax: (addr stream byte) <- lookup *in-data-ah
     926   rewind-stream in-data
     927   var result/eax: boolean <- stream-data-equal? in-data, ",@"
     928   return result
     929 }
     930
     931 fn open-paren-token? _self: (addr token) -> _/eax: boolean {
     932   var self/eax: (addr token) <- copy _self
     933   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     934   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
     935   var in-data/ecx: (addr stream byte) <- copy _in-data
     936   rewind-stream in-data
     937   var g/eax: grapheme <- read-grapheme in-data
     938   compare g, 0x28/open-paren
     939   {
     940     break-if-!=
     941     var result/eax: boolean <- stream-empty? in-data
     942     return result
     943   }
     944   return 0/false
     945 }
     946
     947 fn close-paren-token? _self: (addr token) -> _/eax: boolean {
     948   var self/eax: (addr token) <- copy _self
     949   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     950   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
     951   var in-data/ecx: (addr stream byte) <- copy _in-data
     952   rewind-stream in-data
     953   var g/eax: grapheme <- read-grapheme in-data
     954   compare g, 0x29/close-paren
     955   {
     956     break-if-!=
     957     var result/eax: boolean <- stream-empty? in-data
     958     return result
     959   }
     960   return 0/false
     961 }
     962
     963 fn dot-token? _self: (addr token) -> _/eax: boolean {
     964   var self/eax: (addr token) <- copy _self
     965   var in-data-ah/eax: (addr handle stream byte) <- get self, text-data
     966   var _in-data/eax: (addr stream byte) <- lookup *in-data-ah
     967   var in-data/ecx: (addr stream byte) <- copy _in-data
     968   rewind-stream in-data
     969   var g/eax: grapheme <- read-grapheme in-data
     970   compare g, 0x2e/dot
     971   {
     972     break-if-!=
     973     var result/eax: boolean <- stream-empty? in-data
     974     return result
     975   }
     976   return 0/false
     977 }
     978
     979 fn test-dot-token {
     980   var tmp-storage: (handle token)
     981   var tmp-ah/eax: (addr handle token) <- address tmp-storage
     982   allocate-token tmp-ah
     983   var tmp/eax: (addr token) <- lookup *tmp-ah
     984   initialize-token tmp, "."
     985   var result/eax: boolean <- dot-token? tmp
     986   check result, "F - test-dot-token"
     987 }
     988
     989 fn stream-token? _self: (addr token) -> _/eax: boolean {
     990   var self/eax: (addr token) <- copy _self
     991   var in-type/eax: (addr int) <- get self, type
     992   compare *in-type, 1/stream
     993   {
     994     break-if-=
     995     return 0/false
     996   }
     997   return 1/true
     998 }
     999
    1000 fn skip-token? _self: (addr token) -> _/eax: boolean {
    1001   var self/eax: (addr token) <- copy _self
    1002   var in-type/eax: (addr int) <- get self, type
    1003   compare *in-type, 2/skip
    1004   {
    1005     break-if-=
    1006     return 0/false
    1007   }
    1008   return 1/true
    1009 }
    1010
    1011 fn indent-token? _self: (addr token) -> _/eax: boolean {
    1012   var self/eax: (addr token) <- copy _self
    1013   var in-type/eax: (addr int) <- get self, type
    1014   compare *in-type, 3/indent
    1015   {
    1016     break-if-=
    1017     return 0/false
    1018   }
    1019   return 1/true
    1020 }
    1021
    1022 fn allocate-token _self-ah: (addr handle token) {
    1023   var self-ah/eax: (addr handle token) <- copy _self-ah
    1024   allocate self-ah
    1025   var self/eax: (addr token) <- lookup *self-ah
    1026   var dest-ah/eax: (addr handle stream byte) <- get self, text-data
    1027   populate-stream dest-ah, 0x40/max-symbol-size
    1028 }
    1029
    1030 fn initialize-token _self: (addr token), val: (addr array byte) {
    1031   var self/eax: (addr token) <- copy _self
    1032   var dest-ah/eax: (addr handle stream byte) <- get self, text-data
    1033   populate-stream dest-ah, 0x40
    1034   var dest/eax: (addr stream byte) <- lookup *dest-ah
    1035   write dest, val
    1036 }
    1037
    1038 fn initialize-skip-token _self: (addr token) {
    1039   var self/eax: (addr token) <- copy _self
    1040   var self-type/eax: (addr int) <- get self, type
    1041   copy-to *self-type, 2/skip
    1042 }
    1043
    1044 fn write-token-text-data out: (addr stream byte), _self: (addr token) {
    1045   var self/eax: (addr token) <- copy _self
    1046   var data-ah/eax: (addr handle stream byte) <- get self, text-data
    1047   var data/eax: (addr stream byte) <- lookup *data-ah
    1048   rewind-stream data
    1049   write-stream out, data
    1050 }
    1051
    1052 fn tokens-equal? _a: (addr token), _b: (addr token) -> _/eax: boolean {
    1053   var a/edx: (addr token) <- copy _a
    1054   var b/ebx: (addr token) <- copy _b
    1055   var a-type-addr/eax: (addr int) <- get a, type
    1056   var a-type/eax: int <- copy *a-type-addr
    1057   var b-type-addr/ecx: (addr int) <- get b, type
    1058   compare a-type, *b-type-addr
    1059   {
    1060     break-if-=
    1061     return 0/false
    1062   }
    1063   compare a-type, 2/skip
    1064   {
    1065     break-if-!=
    1066     # skip tokens have no other data
    1067     return 1/true
    1068   }
    1069   compare a-type, 3/indent
    1070   {
    1071     break-if-!=
    1072     # indent tokens have no other data
    1073     var a-number-data-addr/eax: (addr int) <- get a, number-data
    1074     var a-number-data/eax: int <- copy *a-number-data-addr
    1075     var b-number-data-addr/ecx: (addr int) <- get b, number-data
    1076     compare a-number-data, *b-number-data-addr
    1077     {
    1078       break-if-=
    1079       return 0/false
    1080     }
    1081     return 1/true
    1082   }
    1083   var b-data-ah/eax: (addr handle stream byte) <- get b, text-data
    1084   var _b-data/eax: (addr stream byte) <- lookup *b-data-ah
    1085   var b-data/ebx: (addr stream byte) <- copy _b-data
    1086   var a-data-ah/eax: (addr handle stream byte) <- get a, text-data
    1087   var a-data/eax: (addr stream byte) <- lookup *a-data-ah
    1088   var data-match?/eax: boolean <- streams-data-equal? a-data, b-data
    1089   return data-match?
    1090 }
    1091
    1092 fn dump-token-from-cursor _t: (addr token) {
    1093   var t/esi: (addr token) <- copy _t
    1094   var type/eax: (addr int) <- get t, type
    1095   draw-int32-decimal-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, *type, 7/fg 0/bg
    1096   draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, " ", 7/fg 0/bg
    1097   var text-ah/eax: (addr handle stream byte) <- get t, text-data
    1098   var text/eax: (addr stream byte) <- lookup *text-ah
    1099   rewind-stream text
    1100   draw-stream-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, text, 7/fg 0/bg
    1101   draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, " ", 7/fg 0/bg
    1102   var num/eax: (addr int) <- get t, number-data
    1103   draw-int32-decimal-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, *num, 7/fg 0/bg
    1104   draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, "\n", 7/fg 0/bg
    1105 }