Put location data in tokens

lib/pigeon/lexer.rb

@@ -96,51 +96,51 @@ module Pigeon
 
     # This might be a mistake or uneccessary. NN 20 MAR 2020
     def maybe_end_message!
-      if tokens.last.last != :MESSAGE_END
-        @tokens << [:MESSAGE_END]
-        @last_good = :MESSAGE_END
+      if tokens.last.last != :MESSAGE_DELIM
+        @tokens << [:MESSAGE_DELIM, scanner.pos]
+        @last_good = :MESSAGE_DELIM
       end
     end
 
     def do_header
       if scanner.scan(AUTHOR)
         author = scanner.matched.chomp.gsub("author ", "")
-        @tokens << [:AUTHOR, author]
+        @tokens << [:AUTHOR, author, scanner.pos]
         @last_good = :AUTHOR
         return
       end
 
       if scanner.scan(DEPTH)
         depth = scanner.matched.chomp.gsub("depth ", "").to_i
-        @tokens << [:DEPTH, depth]
+        @tokens << [:DEPTH, depth, scanner.pos]
         @last_good = :DEPTH
         return
       end
 
       if scanner.scan(LIPMAA)
         depth = scanner.matched.chomp.gsub("lipmaa ", "")
-        @tokens << [:LIPMAA, depth]
+        @tokens << [:LIPMAA, depth, scanner.pos]
         @last_good = :LIPMAA
         return
       end
 
       if scanner.scan(PREV)
         prev = scanner.matched.chomp.gsub("prev ", "")
-        @tokens << [:PREV, prev]
+        @tokens << [:PREV, prev, scanner.pos]
         @last_good = :PREV
         return
       end
 
       if scanner.scan(KIND)
         kind = scanner.matched.chomp.gsub("kind ", "")
-        @tokens << [:KIND, kind]
+        @tokens << [:KIND, kind, scanner.pos]
         @last_good = :KIND
         return
       end
 
       if scanner.scan(SEPERATOR)
         @state = BODY
-        @tokens << [:HEADER_END]
+        @tokens << [:HEADER_END, scanner.pos]
         @last_good = :HEADER_SEPERATOR
         return
       end
@@ -151,14 +151,14 @@ module Pigeon
     def do_body
       if scanner.scan(BODY_ENTRY)
         key, value = scanner.matched.chomp.split(":")
-        @tokens << [:BODY_ENTRY, key, value]
+        @tokens << [:BODY_ENTRY, key, value, scanner.pos]
         @last_good = :A_BODY_ENTRY
         return
       end
 
       if scanner.scan(SEPERATOR)
         @state = FOOTER
-        @tokens << [:BODY_END]
+        @tokens << [:BODY_END, scanner.pos]
         @last_good = :BODY_SEPERATOR
         return
       end
@@ -172,7 +172,7 @@ module Pigeon
 
       if scanner.scan(FOOTER_ENTRY)
         sig = scanner.matched.strip.gsub("signature ", "")
-        @tokens << [:SIGNATURE, sig]
+        @tokens << [:SIGNATURE, sig, scanner.pos]
         @last_good = :FOOTER_ENTRY
         return
       end

lib/pigeon/parser.rb

@@ -18,16 +18,16 @@ module Pigeon
     def parse
       @tokens.each_with_index do |token, _i|
         case token.first
-        when :AUTHOR then set(:author, token.last)
-        when :KIND then set(:kind, token.last)
-        when :DEPTH then set(:depth, token.last)
-        when :PREV then set(:prev, token.last)
-        when :LIPMAA then set(:lipmaa, token.last)
+        when :AUTHOR then set(:author, token[1])
+        when :KIND then set(:kind, token[1])
+        when :DEPTH then set(:depth, token[1])
+        when :PREV then set(:prev, token[1])
+        when :LIPMAA then set(:lipmaa, token[1])
         when :HEADER_END then set(:body, {})
         when :BODY_ENTRY then set(token[1], token[2], @scratchpad[:body])
         when :BODY_END then nil
-        when :SIGNATURE then set(:signature, token.last)
-        when :MESSAGE_END then finish_this_message!
+        when :SIGNATURE then set(:signature, token[1])
+        when :MESSAGE_DELIM then finish_this_message!
         end
       end
       @results

spec/pigeon/lexer_spec.rb

@@ -2,106 +2,106 @@ require "spec_helper"
 
 RSpec.describe Pigeon::Lexer do
   EXPECTED_TOKENS1 = [
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "f2461a25-6332-4115-b530-91b9ef26b6a5"],
-    [:PREV, "NONE"],
-    [:DEPTH, 0],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"123\""],
-    [:BODY_END],
-    [:SIGNATURE, "BSSVYP4STBKGMA0TVRH9PR038Q3T3KXP8ZTADZJV1BWERQ056Y0T1EAA54RYX009CGYP5PZ3VRRQ2HQTMRGY8G2A0WZ3TNB2V733Y0G.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "20a21e41-859f-4cec-acb4-56a466b93195"],
-    [:PREV, "%R60V14PWNJKG7RKKH04P58SJZKGHR267RTDDHPPGFZ3P98K3HFWG.sha256"],
-    [:DEPTH, 1],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"123\""],
-    [:BODY_END],
-    [:SIGNATURE, "JEXF1C0Y6XGPTAT8EB159J8QYKFNY49ZRW07P5B2ESAQ90V2V6WYEF95SN4C05YAZQZ151ZH4WMRP0BXHZVS1400X3BKBGR0HXRT43G.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "f4cdc570-b49d-487b-8b55-bc85439f3373"],
-    [:PREV, "%VATGV6Q1WXRX330AD6D5PADV84PX2PM9B0BN0CFQY0HWB2DQEGK0.sha256"],
-    [:DEPTH, 2],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"bar\""],
-    [:BODY_END],
-    [:SIGNATURE, "4ZVKC8JMY6DPFVXRSE0W8VJZ0KNF1KMZFCT1BENZ80MNY5AMKF7865E9B2EDY1PCG57EYXRTFJY1BXZESDMY0X7R71T5H52Q0CC8C1R.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "75630163-0ee1-4e3f-8637-0b72f3e2385c"],
-    [:PREV, "%9YF2XH84CA8TSC7NFRVWT5DK8HA67QWTCJBMHYW0W8SX2JXEDN30.sha256"],
-    [:DEPTH, 3],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"bar\""],
-    [:BODY_END],
-    [:SIGNATURE, "AXJPY0QFFACTVWCECN4WQKKKBMH9TA47FXS26FN8WZK4ADWCSE4WNGE5Z75CHRP64PQVW5QRT7V8DXET5E006CMZNDRPDMCC9Y2TT1G.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "9bf8f1d0-3ce1-4092-ba87-8d019dff05b2"],
-    [:PREV, "%ZPR5QZK2FNPCC2XEY4ET1RE7VT3ZE0ZWSY2W4R5DCXPZ7F9C62YG.sha256"],
-    [:DEPTH, 4],
-    [:LIPMAA, "%VATGV6Q1WXRX330AD6D5PADV84PX2PM9B0BN0CFQY0HWB2DQEGK0.sha256"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"123\""],
-    [:BODY_END],
-    [:SIGNATURE, "GXNXBCKNXTE3Z1T6Y3PV6FEWVM3YM3DP86YZRTYFC5AFM4Z2TN5ZKKC0YMT29DZ1X1TJSC3Y5PV6B2RN54ZCKK7ZNPCBA5N6W6WWP38.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "c5abeb0a-7a2e-433c-82ea-2e5cb8246cd9"],
-    [:PREV, "%9JQ1SDJERFXX5SF049B4Y55MPP2PVYG9BZSZRS2W0BMP26PX0V30.sha256"],
-    [:DEPTH, 5],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"123\""],
-    [:BODY_END],
-    [:SIGNATURE, "T20VH1V229ANXB22RJFHWTANKEC9S1ZZEH72A5C3SGRNKH28F0CVR78093YKZXW3645CW0AWFG3FNRYGMBZZ6EC7CMGS3B8NW0J0G28.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "e005d271-42c2-4615-bead-131ce0443e44"],
-    [:PREV, "%JREA65ZVF9K60STANEWW83YW9XZPHDYTX0BESJTHJAEW7HXW8RF0.sha256"],
-    [:DEPTH, 6],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"bar\""],
-    [:BODY_END],
-    [:SIGNATURE, "1S46QAB4RBRN895WWRNDKA3VQ1WS95VEM155X9P8XD1DJ8S0GJBRDJRN5VHNQV5EDMW0APTXQJ28QPPJK3R03K1CH2KV0KF1V5ZA40R.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "6fe0bd91-de26-4588-bb23-5950c0b654f4"],
-    [:PREV, "%Q8Q5QZDSQW5524QC42WDP8GX4G60RJTN866EDV45C0R33RDJ9770.sha256"],
-    [:DEPTH, 7],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"bar\""],
-    [:BODY_END],
-    [:SIGNATURE, "YE2Y4K70BKX1Q7HPBEHH3Y6C4DW577E4AR1CPYF62KXHC81EREFM2TF4JX80CR92J9HJ8JV4CMP7TAB1WF16KA158344W2W59BTDA1G.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "8cf7242b-c2ec-4c06-85fc-4294bb9881f9"],
-    [:PREV, "%V9K764RRC78D95AT5H7Y1GWN4S9VTFS82YWD0NRYQT51R44DZ630.sha256"],
-    [:DEPTH, 8],
-    [:LIPMAA, "%9JQ1SDJERFXX5SF049B4Y55MPP2PVYG9BZSZRS2W0BMP26PX0V30.sha256"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"123\""],
-    [:BODY_END],
-    [:SIGNATURE, "Q61K5HK1WE03ST965Z9X027WSDGE9PFRBYYQT40T4N582ZN46KJTFC9Z92BZC90QFNY37BZ0ZBH6S86KNV371MNMWG4CT2Q016XT430.sig.ed25519"],
-    [:MESSAGE_END],
-    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519"],
-    [:KIND, "caadfdee-0600-44cf-befd-6332c067a3e7"],
-    [:PREV, "%ZRZRSNHAYA10JS3PMNJHNJAQJ1EV1KF9KEATD7JMZ2H78XKFEZXG.sha256"],
-    [:DEPTH, 9],
-    [:LIPMAA, "NONE"],
-    [:HEADER_END],
-    [:BODY_ENTRY, "foo", "\"bar\""],
-    [:BODY_END],
-    [:SIGNATURE, "9TJX6KF51GQNF2J05KPB4V7WZBMKVRV1ECNC6W7ETN4ZQ636DCPH1C95C1W19ZTXMB53TSQK2MW3PFRBKN05C8F8PNY0PG7WG3Q2T30.sig.ed25519"],
-    [:MESSAGE_END],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 69],
+    [:KIND, "f2461a25-6332-4115-b530-91b9ef26b6a5", 111],
+    [:PREV, "NONE", 121],
+    [:DEPTH, 0, 129],
+    [:LIPMAA, "NONE", 141],
+    [:HEADER_END, 142],
+    [:BODY_ENTRY, "foo", "\"123\"", 152],
+    [:BODY_END, 153],
+    [:SIGNATURE, "BSSVYP4STBKGMA0TVRH9PR038Q3T3KXP8ZTADZJV1BWERQ056Y0T1EAA54RYX009CGYP5PZ3VRRQ2HQTMRGY8G2A0WZ3TNB2V733Y0G.sig.ed25519", 279],
+    [:MESSAGE_DELIM, 280],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 349],
+    [:KIND, "20a21e41-859f-4cec-acb4-56a466b93195", 391],
+    [:PREV, "%R60V14PWNJKG7RKKH04P58SJZKGHR267RTDDHPPGFZ3P98K3HFWG.sha256", 457],
+    [:DEPTH, 1, 465],
+    [:LIPMAA, "NONE", 477],
+    [:HEADER_END, 478],
+    [:BODY_ENTRY, "foo", "\"123\"", 488],
+    [:BODY_END, 489],
+    [:SIGNATURE, "JEXF1C0Y6XGPTAT8EB159J8QYKFNY49ZRW07P5B2ESAQ90V2V6WYEF95SN4C05YAZQZ151ZH4WMRP0BXHZVS1400X3BKBGR0HXRT43G.sig.ed25519", 615],
+    [:MESSAGE_DELIM, 616],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 685],
+    [:KIND, "f4cdc570-b49d-487b-8b55-bc85439f3373", 727],
+    [:PREV, "%VATGV6Q1WXRX330AD6D5PADV84PX2PM9B0BN0CFQY0HWB2DQEGK0.sha256", 793],
+    [:DEPTH, 2, 801],
+    [:LIPMAA, "NONE", 813],
+    [:HEADER_END, 814],
+    [:BODY_ENTRY, "foo", "\"bar\"", 824],
+    [:BODY_END, 825],
+    [:SIGNATURE, "4ZVKC8JMY6DPFVXRSE0W8VJZ0KNF1KMZFCT1BENZ80MNY5AMKF7865E9B2EDY1PCG57EYXRTFJY1BXZESDMY0X7R71T5H52Q0CC8C1R.sig.ed25519", 951],
+    [:MESSAGE_DELIM, 952],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 1021],
+    [:KIND, "75630163-0ee1-4e3f-8637-0b72f3e2385c", 1063],
+    [:PREV, "%9YF2XH84CA8TSC7NFRVWT5DK8HA67QWTCJBMHYW0W8SX2JXEDN30.sha256", 1129],
+    [:DEPTH, 3, 1137],
+    [:LIPMAA, "NONE", 1149],
+    [:HEADER_END, 1150],
+    [:BODY_ENTRY, "foo", "\"bar\"", 1160],
+    [:BODY_END, 1161],
+    [:SIGNATURE, "AXJPY0QFFACTVWCECN4WQKKKBMH9TA47FXS26FN8WZK4ADWCSE4WNGE5Z75CHRP64PQVW5QRT7V8DXET5E006CMZNDRPDMCC9Y2TT1G.sig.ed25519", 1287],
+    [:MESSAGE_DELIM, 1288],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 1357],
+    [:KIND, "9bf8f1d0-3ce1-4092-ba87-8d019dff05b2", 1399],
+    [:PREV, "%ZPR5QZK2FNPCC2XEY4ET1RE7VT3ZE0ZWSY2W4R5DCXPZ7F9C62YG.sha256", 1465],
+    [:DEPTH, 4, 1473],
+    [:LIPMAA, "%VATGV6Q1WXRX330AD6D5PADV84PX2PM9B0BN0CFQY0HWB2DQEGK0.sha256", 1541],
+    [:HEADER_END, 1542],
+    [:BODY_ENTRY, "foo", "\"123\"", 1552],
+    [:BODY_END, 1553],
+    [:SIGNATURE, "GXNXBCKNXTE3Z1T6Y3PV6FEWVM3YM3DP86YZRTYFC5AFM4Z2TN5ZKKC0YMT29DZ1X1TJSC3Y5PV6B2RN54ZCKK7ZNPCBA5N6W6WWP38.sig.ed25519", 1679],
+    [:MESSAGE_DELIM, 1680],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 1749],
+    [:KIND, "c5abeb0a-7a2e-433c-82ea-2e5cb8246cd9", 1791],
+    [:PREV, "%9JQ1SDJERFXX5SF049B4Y55MPP2PVYG9BZSZRS2W0BMP26PX0V30.sha256", 1857],
+    [:DEPTH, 5, 1865],
+    [:LIPMAA, "NONE", 1877],
+    [:HEADER_END, 1878],
+    [:BODY_ENTRY, "foo", "\"123\"", 1888],
+    [:BODY_END, 1889],
+    [:SIGNATURE, "T20VH1V229ANXB22RJFHWTANKEC9S1ZZEH72A5C3SGRNKH28F0CVR78093YKZXW3645CW0AWFG3FNRYGMBZZ6EC7CMGS3B8NW0J0G28.sig.ed25519", 2015],
+    [:MESSAGE_DELIM, 2016],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 2085],
+    [:KIND, "e005d271-42c2-4615-bead-131ce0443e44", 2127],
+    [:PREV, "%JREA65ZVF9K60STANEWW83YW9XZPHDYTX0BESJTHJAEW7HXW8RF0.sha256", 2193],
+    [:DEPTH, 6, 2201],
+    [:LIPMAA, "NONE", 2213],
+    [:HEADER_END, 2214],
+    [:BODY_ENTRY, "foo", "\"bar\"", 2224],
+    [:BODY_END, 2225],
+    [:SIGNATURE, "1S46QAB4RBRN895WWRNDKA3VQ1WS95VEM155X9P8XD1DJ8S0GJBRDJRN5VHNQV5EDMW0APTXQJ28QPPJK3R03K1CH2KV0KF1V5ZA40R.sig.ed25519", 2351],
+    [:MESSAGE_DELIM, 2352],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 2421],
+    [:KIND, "6fe0bd91-de26-4588-bb23-5950c0b654f4", 2463],
+    [:PREV, "%Q8Q5QZDSQW5524QC42WDP8GX4G60RJTN866EDV45C0R33RDJ9770.sha256", 2529],
+    [:DEPTH, 7, 2537],
+    [:LIPMAA, "NONE", 2549],
+    [:HEADER_END, 2550],
+    [:BODY_ENTRY, "foo", "\"bar\"", 2560],
+    [:BODY_END, 2561],
+    [:SIGNATURE, "YE2Y4K70BKX1Q7HPBEHH3Y6C4DW577E4AR1CPYF62KXHC81EREFM2TF4JX80CR92J9HJ8JV4CMP7TAB1WF16KA158344W2W59BTDA1G.sig.ed25519", 2687],
+    [:MESSAGE_DELIM, 2688],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 2757],
+    [:KIND, "8cf7242b-c2ec-4c06-85fc-4294bb9881f9", 2799],
+    [:PREV, "%V9K764RRC78D95AT5H7Y1GWN4S9VTFS82YWD0NRYQT51R44DZ630.sha256", 2865],
+    [:DEPTH, 8, 2873],
+    [:LIPMAA, "%9JQ1SDJERFXX5SF049B4Y55MPP2PVYG9BZSZRS2W0BMP26PX0V30.sha256", 2941],
+    [:HEADER_END, 2942],
+    [:BODY_ENTRY, "foo", "\"123\"", 2952],
+    [:BODY_END, 2953],
+    [:SIGNATURE, "Q61K5HK1WE03ST965Z9X027WSDGE9PFRBYYQT40T4N582ZN46KJTFC9Z92BZC90QFNY37BZ0ZBH6S86KNV371MNMWG4CT2Q016XT430.sig.ed25519", 3079],
+    [:MESSAGE_DELIM, 3080],
+    [:AUTHOR, "@5W9Z3TB8K8F0D6WKCRYZKRAKRWXN5YXHHNEZ4EYDWBWX0CGY9SZ0.ed25519", 3149],
+    [:KIND, "caadfdee-0600-44cf-befd-6332c067a3e7", 3191],
+    [:PREV, "%ZRZRSNHAYA10JS3PMNJHNJAQJ1EV1KF9KEATD7JMZ2H78XKFEZXG.sha256", 3257],
+    [:DEPTH, 9, 3265],
+    [:LIPMAA, "NONE", 3277],
+    [:HEADER_END, 3278],
+    [:BODY_ENTRY, "foo", "\"bar\"", 3288],
+    [:BODY_END, 3289],
+    [:SIGNATURE, "9TJX6KF51GQNF2J05KPB4V7WZBMKVRV1ECNC6W7ETN4ZQ636DCPH1C95C1W19ZTXMB53TSQK2MW3PFRBKN05C8F8PNY0PG7WG3Q2T30.sig.ed25519", 3415],
+    [:MESSAGE_DELIM, 3415],
   ].freeze
 
   MESSAGE_LINES = [
@@ -141,12 +141,12 @@ RSpec.describe Pigeon::Lexer do
     tokens = Pigeon::Lexer.tokenize(string)
     hash = tokens.each_with_object({ BODY: {} }) do |token, h|
       case token.first
-      when :HEADER_END, :BODY_END, :MESSAGE_END
+      when :HEADER_END, :BODY_END, :MESSAGE_DELIM
         h
       when :BODY_ENTRY
         h[:BODY][token[1]] = token[2]
       else
-        h[token.first] = token.last
+        h[token.first] = token[1]
       end
     end
 

spec/pigeon/message_spec.rb

@@ -140,7 +140,7 @@ RSpec.describe Pigeon::Message do
       [:HEADER_END],
       [:BODY_ENTRY, "duplicate", "This key is a duplicate."],
       [:SIGNATURE, "DN7yPTE-m433ND3jBL4oM23XGxBKafjq0Dp9ArBQa_TIGU7DmCxTumieuPBN-NKxlx_0N7-c5zjLb5XXVHYPCQ==.sig.ed25519"],
-      [:MESSAGE_END],
+      [:MESSAGE_DELIM],
     ]
     e = Pigeon::Helpers::VerificationError
     m = "Expected field `depth` to equal 0, got: 10"

spec/pigeon/parser_spec.rb

@@ -19,7 +19,7 @@ RSpec.describe Pigeon::Lexer do
     [:BODY_ENTRY, "duplicate", "Pigeon does not allow duplicate keys."],
     [:BODY_ENTRY, "duplicate", "This key is a duplicate."],
     [:SIGNATURE, "DN7yPTE-m433ND3jBL4oM23XGxBKafjq0Dp9ArBQa_TIGU7DmCxTumieuPBN-NKxlx_0N7-c5zjLb5XXVHYPCQ==.sig.ed25519"],
-    [:MESSAGE_END],
+    [:MESSAGE_DELIM],
   ].freeze
 
   it "parses tokens" do