Move Pigeon::Lexer into its own module

README.md

@@ -37,6 +37,7 @@ Eg: `pigeon identity show` becomes `./pigeon-cli show`.
  - [X] pigeon message find-all for local feed.
  - [ ] pigeon bundle consume
  - [ ] Change all the `{40,90}` values in ::Lexer to real length values
+ - [ ] Rename `message find` to `message read`, since other finders return a multihash.
  - [ ] Create regexes in ::Lexer using strings and Regexp.new() for cleaner regexes.
  - [ ] pigeon message find-all for peer feed. I will need to add index for `author => message_count`
  - [ ] refactor `Bundle.create` to use `message find-all`.

dist/pigeon.rb

@@ -60,4 +60,5 @@ require_relative File.join("pigeon", "draft_serializer.rb")
 require_relative File.join("pigeon", "message_serializer.rb")
 require_relative File.join("pigeon", "message.rb")
 require_relative File.join("pigeon", "draft.rb")
+require_relative File.join("pigeon", "lexer.rb")
 require_relative File.join("pigeon", "bundle.rb")

dist/pigeon/bundle.rb

@@ -1,72 +1,5 @@
 module Pigeon
   class Bundle
-    class Lexer
-      # No *_VALUE can be > 128 chars.
-      MAX_CHUNK_SIZE = 128
-      # TODO: Change all the `{40,90}` values in ::Lexer to real values
-      # TODO: Create regexes using string and Regexp.new() for cleaner regexes.
-      FEED_VALUE = /@.{40,90}.ed25519/
-      DEPTH_COUNT = /\d{1,7}/
-      MESG_VALUE = /%.{40,90}.sha256/
-      BLOB_VALUE = /&.{40,90}.sha256/
-      NULL_VALUE = /NONE/
-      STRG_VALUE = /".{1,64}"/
-      ALPHANUMERICISH = /[a-zA-Z\d\.]{1,64}/
-      ALL_VALUES = [
-        FEED_VALUE,
-        MESG_VALUE,
-        NULL_VALUE,
-        STRG_VALUE,
-        BLOB_VALUE,
-      ].map(&:source).join("|")
-      ANY_VALUE = Regexp.new(ALL_VALUES)
-
-      SEPERATOR = /\n\n/
-      AUTHOR = /author #{FEED_VALUE}/
-      DEPTH = /depth #{DEPTH_COUNT}/
-      PREV = /prev (#{MESG_VALUE}|#{NULL_VALUE})/
-      KIND = /kind #{ALPHANUMERICISH}/
-      BODY_ENTRY = /#{ALPHANUMERICISH}:#{ANY_VALUE}\n/
-
-      FOOTER = /signature .*{40,90}\.sig\.ed25519/
-
-      def self.tokenize(bundle_string)
-        new.tokenize(bundle_string)
-      end
-
-      def stack
-        @stack ||= []
-      end
-
-      def state
-        stack.last || :header
-      end
-
-      def push_state(state)
-        stack.push(state)
-      end
-
-      def pop_state
-        stack.pop
-      end
-
-      def scan_header(scanner)
-      end
-
-      def tokenize(bundle_string)
-        scanner = StringScanner.new(bundle_string)
-        tokens = []
-        until scanner.eos?
-          case state
-          when :header
-            raise "WIP"
-          else
-            raise "Bad state?"
-          end
-        end
-      end
-    end
-
     def self.create(file_path = DEFAULT_BUNDLE_PATH)
       s = Pigeon::Storage.current
       last = s.message_count
@@ -85,12 +18,6 @@ module Pigeon
 
     private
 
-    class Parser
-    end
-
-    class Interpreter
-    end
-
     def initialize
     end
   end

dist/pigeon/lexer.rb

@@ -0,0 +1,91 @@
+module Pigeon
+  class Lexer
+    attr_reader :bundle_string, :scanner, :tokens
+    # TODO: Change all the `{40,90}` values in ::Lexer to real values
+    # TODO: Create regexes using string and Regexp.new() for cleaner regexes.
+    FEED_VALUE = /@.{40,90}.ed25519/
+    DEPTH_COUNT = /\d{1,7}/
+    MESG_VALUE = /%.{40,90}.sha256/
+    BLOB_VALUE = /&.{40,90}.sha256/
+    NULL_VALUE = /NONE/
+    STRG_VALUE = /".{1,64}"/
+    ALPHANUMERICISH = /[a-zA-Z\d\.]{1,64}/
+    ALL_VALUES = [
+      FEED_VALUE,
+      MESG_VALUE,
+      NULL_VALUE,
+      STRG_VALUE,
+      BLOB_VALUE,
+    ].map(&:source).join("|")
+    ANY_VALUE = Regexp.new(ALL_VALUES)
+
+    SEPERATOR = /\n\n/
+    AUTHOR = /author #{FEED_VALUE}/
+    DEPTH = /depth #{DEPTH_COUNT}/
+    PREV = /prev (#{MESG_VALUE}|#{NULL_VALUE})/
+    KIND = /kind #{ALPHANUMERICISH}/
+    BODY_ENTRY = /#{ALPHANUMERICISH}:#{ANY_VALUE}\n/
+
+    FOOTER_ENTRY = /signature .*{40,90}\.sig\.ed25519/
+
+    LEXER_STATES = [HEADER = :header, BODY = :body, FOOTER = :footer]
+
+    def self.tokenize(bundle_string)
+      # TODO: Maybe move #tokeinze into constructor.
+      new(bundle_string).tokenize
+    end
+
+    def initialize(bundle_string)
+      @bundle_string = bundle_string
+      @scanner = StringScanner.new(bundle_string)
+      @tokens = []
+    end
+
+    def stack
+      @stack ||= []
+    end
+
+    def state
+      stack.last || :header
+    end
+
+    def push_state(state)
+      stack.push(state)
+    end
+
+    def pop_state
+      stack.pop
+    end
+
+    def scan_header(scanner)
+    end
+
+    def do_header
+      if scanner.scan(WHATEVER)
+        tokens << [:OPEN_BLOCK]
+        push_state :expression
+        return
+      end
+
+      if scanner.scan_until(/.*?(?={{)/m)
+        tokens << [:CONTENT, scanner.matched]
+      else
+        tokens << [:CONTENT, scanner.rest]
+        scanner.terminate
+      end
+    end
+
+    def tokenize
+      until scanner.eos?
+        case state
+        when HEADER
+          raise "WIP"
+        when BODY
+          raise "WIP"
+        when FOOTER
+          raise "WIP"
+        end
+      end
+    end
+  end
+end

spec/pigeon/lexer_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
-RSpec.describe Pigeon::Bundle::Lexer do
+RSpec.describe Pigeon::Lexer do
   it "tokenizes the bundle" do
-    Pigeon::Bundle::Lexer.tokenize(File.read("./example.bundle"))
+    Pigeon::Lexer.tokenize(File.read("./example.bundle"))
   end
 end