https://github.com/akkartik/mu/blob/main/linux/403unicode.mu

      1 # Helpers for Unicode.
      2 #
      3 # Mu has no characters, only code points and graphemes.
      4 # Code points are the indivisible atoms of text streams.
      5 #   https://en.wikipedia.org/wiki/Code_point
      6 # Graphemes are the smallest self-contained unit of text.
      7 # Graphemes may consist of multiple code points.
      8 #
      9 # Mu graphemes are always represented in utf-8, and they are required to fit
     10 # in 4 bytes.
     11 #
     12 # Mu doesn't currently support combining code points, or graphemes made of
     13 # multiple code points. One day we will.
     14 # We also don't currently support code points that translate into multiple
     15 # or wide graphemes. (In particular, Tab will never be supported.)
     16
     17 # transliterated from tb_utf8_unicode_to_char in https://github.com/nsf/termbox
     18 # https://wiki.tcl-lang.org/page/UTF%2D8+bit+by+bit explains the algorithm
     19 #
     20 # The day we want to support combining characters, this function will need to
     21 # take multiple code points. Or something.
     22 fn to-grapheme in: code-point -> _/eax: grapheme {
     23   var c/eax: int <- copy in
     24   var num-trailers/ecx: int <- copy 0
     25   var first/edx: int <- copy 0
     26   $to-grapheme:compute-length: {
     27     # single byte: just return it
     28     compare c, 0x7f
     29     {
     30       break-if->
     31       var g/eax: grapheme <- copy c
     32       return g
     33     }
     34     # 2 bytes
     35     compare c, 0x7ff
     36     {
     37       break-if->
     38       num-trailers <- copy 1
     39       first <- copy 0xc0
     40       break $to-grapheme:compute-length
     41     }
     42     # 3 bytes
     43     compare c, 0xffff
     44     {
     45       break-if->
     46       num-trailers <- copy 2
     47       first <- copy 0xe0
     48       break $to-grapheme:compute-length
     49     }
     50     # 4 bytes
     51     compare c, 0x1fffff
     52     {
     53       break-if->
     54       num-trailers <- copy 3
     55       first <- copy 0xf0
     56       break $to-grapheme:compute-length
     57     }
     58     # more than 4 bytes: unsupported
     59     # TODO: print to stderr
     60     compare c, 0x1fffff
     61     {
     62       break-if->
     63       print-string-to-real-screen "unsupported code point "
     64       print-int32-hex-to-real-screen c
     65       print-string-to-real-screen "\n"
     66       var exit-status/ebx: int <- copy 1
     67       syscall_exit
     68     }
     69   }
     70   # emit trailer bytes, 6 bits from 'in', first two bits '10'
     71   var result/edi: grapheme <- copy 0
     72   {
     73     compare num-trailers, 0
     74     break-if-<=
     75     var tmp/esi: int <- copy c
     76     tmp <- and 0x3f
     77     tmp <- or 0x80
     78     result <- shift-left 8
     79     result <- or tmp
     80     # update loop state
     81     c <- shift-right 6
     82     num-trailers <- decrement
     83     loop
     84   }
     85   # emit engine
     86   result <- shift-left 8
     87   result <- or c
     88   result <- or first
     89   #
     90   return result
     91 }
     92
     93 # single-byte code point have identical graphemes
     94 fn test-to-grapheme-single-byte {
     95   var in-int/ecx: int <- copy 0
     96   {
     97     compare in-int, 0x7f
     98     break-if->
     99     var in/eax: code-point <- copy in-int
    100     var out/eax: grapheme <- to-grapheme in
    101     var out-int/eax: int <- copy out
    102     check-ints-equal out-int, in-int, "F - test-to-grapheme-single-byte"
    103     in-int <- increment
    104     loop
    105   }
    106 }
    107
    108                                                               # byte       | byte      | byte      | byte
    109 # smallest 2-byte utf-8
    110 fn test-to-grapheme-two-bytes-min {
    111   var in/eax: code-point <- copy 0x80                         #                                 10     00-0000
    112   var out/eax: grapheme <- to-grapheme in
    113   var out-int/eax: int <- copy out
    114   check-ints-equal out-int, 0x80c2, "F - to-grapheme/2a"      #                         110 0-0010  10 00-0000
    115 }
    116
    117 # largest 2-byte utf-8
    118 fn test-to-grapheme-two-bytes-max {
    119   var in/eax: code-point <- copy 0x7ff                        #                             1-1111     11-1111
    120   var out/eax: grapheme <- to-grapheme in
    121   var out-int/eax: int <- copy out
    122   check-ints-equal out-int, 0xbfdf, "F - to-grapheme/2b"      #                         110 1-1111  10 11-1111
    123 }
    124
    125 # smallest 3-byte utf-8
    126 fn test-to-grapheme-three-bytes-min {
    127   var in/eax: code-point <- copy 0x800                        #                            10-0000     00-0000
    128   var out/eax: grapheme <- to-grapheme in
    129   var out-int/eax: int <- copy out
    130   check-ints-equal out-int, 0x80a0e0, "F - to-grapheme/3a"    #              1110 0000  10 10-0000  10 00-0000
    131 }
    132
    133 # largest 3-byte utf-8
    134 fn test-to-grapheme-three-bytes-max {
    135   var in/eax: code-point <- copy 0xffff                       #                   1111     11-1111     11-1111
    136   var out/eax: grapheme <- to-grapheme in
    137   var out-int/eax: int <- copy out
    138   check-ints-equal out-int, 0xbfbfef, "F - to-grapheme/3b"    #              1110 1111  10 11-1111  10 11-1111
    139 }
    140
    141 # smallest 4-byte utf-8
    142 fn test-to-grapheme-four-bytes-min {
    143   var in/eax: code-point <- copy 0x10000                      #                 1-0000     00-0000     00-0000
    144   var out/eax: grapheme <- to-grapheme in
    145   var out-int/eax: int <- copy out
    146   check-ints-equal out-int, 0x808090f0, "F - to-grapheme/4a"  # 1111-0 000  10 01-0000  10 00-0000  10 00-0000
    147 }
    148
    149 # largest 4-byte utf-8
    150 fn test-to-grapheme-four-bytes-max {
    151   var in/eax: code-point <- copy 0x1fffff                     #        111     11-1111     11-1111     11-1111
    152   var out/eax: grapheme <- to-grapheme in
    153   var out-int/eax: int <- copy out
    154   check-ints-equal out-int, 0xbfbfbff7, "F - to-grapheme/4b"  # 1111-0 111  10 11-1111  10 11-1111  10 11-1111
    155 }
    156
    157 # read the next grapheme from a stream of bytes
    158 fn read-grapheme in: (addr stream byte) -> _/eax: grapheme {
    159   # if at eof, return EOF
    160   {
    161     var eof?/eax: boolean <- stream-empty? in
    162     compare eof?, 0/false
    163     break-if-=
    164     return 0xffffffff
    165   }
    166   var c/eax: byte <- read-byte in
    167   var num-trailers/ecx: int <- copy 0
    168   $read-grapheme:compute-length: {
    169     # single byte: just return it
    170     compare c, 0xc0
    171     {
    172       break-if->=
    173       var g/eax: grapheme <- copy c
    174       return g
    175     }
    176     compare c, 0xfe
    177     {
    178       break-if-<
    179       var g/eax: grapheme <- copy c
    180       return g
    181     }
    182     # 2 bytes
    183     compare c, 0xe0
    184     {
    185       break-if->=
    186       num-trailers <- copy 1
    187       break $read-grapheme:compute-length
    188     }
    189     # 3 bytes
    190     compare c, 0xf0
    191     {
    192       break-if->=
    193       num-trailers <- copy 2
    194       break $read-grapheme:compute-length
    195     }
    196     # 4 bytes
    197     compare c, 0xf8
    198     {
    199       break-if->=
    200       num-trailers <- copy 3
    201       break $read-grapheme:compute-length
    202     }
    203 $read-grapheme:abort: {
    204       # TODO: print to stderr
    205       print-string-to-real-screen "utf-8 encodings larger than 4 bytes are not supported. First byte seen: "
    206       var n/eax: int <- copy c
    207       print-int32-hex-to-real-screen n
    208       print-string-to-real-screen "\n"
    209       var exit-status/ebx: int <- copy 1
    210       syscall_exit
    211     }
    212   }
    213   # prepend trailer bytes
    214   var result/edi: grapheme <- copy c
    215   var num-byte-shifts/edx: int <- copy 1
    216   {
    217     compare num-trailers, 0
    218     break-if-<=
    219     var tmp/eax: byte <- read-byte in
    220     var tmp2/eax: int <- copy tmp
    221     tmp2 <- shift-left-bytes tmp2, num-byte-shifts
    222     result <- or tmp2
    223     # update loop state
    224     num-byte-shifts <- increment
    225     num-trailers <- decrement
    226     loop
    227   }
    228   return result
    229 }
    230
    231 fn test-read-grapheme {
    232   var s: (stream byte 0x30)
    233   var s2/ecx: (addr stream byte) <- address s
    234   write s2, "aΒc世d界e"
    235   var c/eax: grapheme <- read-grapheme s2
    236   var n/eax: int <- copy c
    237   check-ints-equal n, 0x61, "F - test grapheme/0"
    238   var c/eax: grapheme <- read-grapheme s2
    239   var n/eax: int <- copy c
    240   check-ints-equal n, 0x92ce/greek-capital-letter-beta, "F - test grapheme/1"
    241   var c/eax: grapheme <- read-grapheme s2
    242   var n/eax: int <- copy c
    243   check-ints-equal n, 0x63, "F - test grapheme/2"
    244   var c/eax: grapheme <- read-grapheme s2
    245   var n/eax: int <- copy c
    246   check-ints-equal n, 0x96b8e4, "F - test grapheme/3"
    247   var c/eax: grapheme <- read-grapheme s2
    248   var n/eax: int <- copy c
    249   check-ints-equal n, 0x64, "F - test grapheme/4"
    250   var c/eax: grapheme <- read-grapheme s2
    251   var n/eax: int <- copy c
    252   check-ints-equal n, 0x8c95e7, "F - test grapheme/5"
    253   var c/eax: grapheme <- read-grapheme s2
    254   var n/eax: int <- copy c
    255   check-ints-equal n, 0x65, "F - test grapheme/6"
    256 }
    257
    258 fn read-grapheme-buffered in: (addr buffered-file) -> _/eax: grapheme {
    259   var c/eax: byte <- read-byte-buffered in
    260   var num-trailers/ecx: int <- copy 0
    261   $read-grapheme-buffered:compute-length: {
    262     # single byte: just return it
    263     compare c, 0xc0
    264     {
    265       break-if->=
    266       var g/eax: grapheme <- copy c
    267       return g
    268     }
    269     compare c, 0xfe
    270     {
    271       break-if-<
    272       var g/eax: grapheme <- copy c
    273       return g
    274     }
    275     # 2 bytes
    276     compare c, 0xe0
    277     {
    278       break-if->=
    279       num-trailers <- copy 1
    280       break $read-grapheme-buffered:compute-length
    281     }
    282     # 3 bytes
    283     compare c, 0xf0
    284     {
    285       break-if->=
    286       num-trailers <- copy 2
    287       break $read-grapheme-buffered:compute-length
    288     }
    289     # 4 bytes
    290     compare c, 0xf8
    291     {
    292       break-if->=
    293       num-trailers <- copy 3
    294       break $read-grapheme-buffered:compute-length
    295     }
    296 $read-grapheme-buffered:abort: {
    297       # TODO: print to stderr
    298       print-string-to-real-screen "utf-8 encodings larger than 4 bytes are not supported. First byte seen: "
    299       var n/eax: int <- copy c
    300       print-int32-hex-to-real-screen n
    301       print-string-to-real-screen "\n"
    302       var exit-status/ebx: int <- copy 1
    303       syscall_exit
    304     }
    305   }
    306   # prepend trailer bytes
    307   var result/edi: grapheme <- copy c
    308   var num-byte-shifts/edx: int <- copy 1
    309   {
    310     compare num-trailers, 0
    311     break-if-<=
    312     var tmp/eax: byte <- read-byte-buffered in
    313     var tmp2/eax: int <- copy tmp
    314     tmp2 <- shift-left-bytes tmp2, num-byte-shifts
    315     result <- or tmp2
    316     # update loop state
    317     num-byte-shifts <- increment
    318     num-trailers <- decrement
    319     loop
    320   }
    321   return result
    322 }
    323
    324 # needed because available primitives only shift by a literal/constant number of bits
    325 fn shift-left-bytes n: int, k: int -> _/eax: int {
    326   var i/ecx: int <- copy 0
    327   var result/eax: int <- copy n
    328   {
    329     compare i, k
    330     break-if->=
    331     compare i, 4  # only 4 bytes in 32 bits
    332     break-if->=
    333     result <- shift-left 8
    334     i <- increment
    335     loop
    336   }
    337   return result
    338 }
    339
    340 fn test-shift-left-bytes-0 {
    341   var result/eax: int <- shift-left-bytes 1, 0
    342   check-ints-equal result, 1, "F - shift-left-bytes 0"
    343 }
    344
    345 fn test-shift-left-bytes-1 {
    346   var result/eax: int <- shift-left-bytes 1, 1
    347   check-ints-equal result, 0x100, "F - shift-left-bytes 1"
    348 }
    349
    350 fn test-shift-left-bytes-2 {
    351   var result/eax: int <- shift-left-bytes 1, 2
    352   check-ints-equal result, 0x10000, "F - shift-left-bytes 2"
    353 }
    354
    355 fn test-shift-left-bytes-3 {
    356   var result/eax: int <- shift-left-bytes 1, 3
    357   check-ints-equal result, 0x1000000, "F - shift-left-bytes 3"
    358 }
    359
    360 fn test-shift-left-bytes-4 {
    361   var result/eax: int <- shift-left-bytes 1, 4
    362   check-ints-equal result, 0, "F - shift-left-bytes 4"
    363 }
    364
    365 fn test-shift-left-bytes-5 {
    366   var result/eax: int <- shift-left-bytes 1, 5
    367   check-ints-equal result, 0, "F - shift-left-bytes >4"
    368 }
    369
    370 # write a grapheme to a stream of bytes
    371 # this is like write-to-stream, except we skip leading 0 bytes
    372 fn write-grapheme out: (addr stream byte), g: grapheme {
    373 $write-grapheme:body: {
    374   var c/eax: int <- copy g
    375   append-byte out, c  # first byte is always written
    376   c <- shift-right 8
    377   compare c, 0
    378   break-if-= $write-grapheme:body
    379   append-byte out, c
    380   c <- shift-right 8
    381   compare c, 0
    382   break-if-= $write-grapheme:body
    383   append-byte out, c
    384   c <- shift-right 8
    385   compare c, 0
    386   break-if-= $write-grapheme:body
    387   append-byte out, c
    388 }
    389 }