Update README

README.md

@@ -1,80 +1 @@
-![](logo.png)
-
-# Pigeon Ruby
-
-A [Pigeon Protocol](https://tildegit.org/PigeonProtocolConsortium/protocol_spec) client written in Ruby.
-
-Email `contact` at `vaporsoft.xyz` to ask questions or get involved. Your feedback is solicited and appreciated. Seriously, send us an email! We look forward to hearing from you.
-
-# Features
-
- * CLI (docs via `pigeon-cli help`) and Ruby API available ([docs here](ruby_tutorial.md))
- * Minimal dependencies - only outside deps are `thor` (for CLI) and `ed25519` (for signatures).
- * Thoroughly unit tested.
-
-# Caveats
-
- * Current windows support is unknown (and unlikely to work in current state). Please report bugs.
- * Not published to RubyGems yet (see installation instructions below)
- * Single threaded use is assumed. Built for a single user per OS process. Many design tradeoffs were made around that use case.
- * Bundling operations need performance tuning. Optimizations are planned and help is welcome.
-
-# Build From Source
-
-We are not yet on Rubygems. The gem will be released after we are fully compliant with the spec.
-
-In the meantime:
-
-```
-git clone https://tildegit.org/PigeonProtocolConsortium/pigeon_ruby.git
-cd pigeon_ruby
-gem build pigeon.gemspec
-gem install pigeon-0.1.1.gem
-pigeon-cli identity new # Should work. Raise issue if not.
-pigeon-cli status
-pigeon-cli help
-```
-
-# Usage: CLI
-
-See `pigeon-cli help` for documentation.
-See `kitchen_sink.sh` examples.
-
-# Usage: Ruby Lib
-
-[Docs available here](ruby_tutorial.md)
-
-# Current Status
-
- - [ ] CLI is wrapping `FILE.` and `FEED.` multihahshes in "string quotes". Why?
- - [ ] Update Dev docs in protocol spec to reflect changes to `lipmaa` header.
- - [ ] Update spec document CLI usage examples to reflect API changes in 2020.
- - [ ] 100% class / module documentation
- - [ ] Run a [terminology extraction tool](https://www.visualthesaurus.com/vocabgrabber/#) on the documentation and write a glossary of terms.
- - [ ] Publish to RubyGems
-
-# Optimizations
-
- - [ ] add parsers and validators for all CLI inputs
- - [ ] Make the switch to LevelDB, RocksDB, [UNQLite](https://unqlite.org/features.html) or similar (currently using Ruby PStore).
- - [ ] Reduce whole darn repo into single module to aide portability. `::Helpers` module is OK.
- - [ ] Update the bundles.md document once `bundle consume` works.
- - [ ] Performance benchmarks (Do this second to last!)
- - [ ] Performance tuning (Do this last!)
-
-# New Features / Road Map
-
- - [ ] Support partial verification via `lipmaa` property.
- - [ ] Add `--since=`/`--until=` args to `bundle create` for sending partial / "slice" bundles.
- - [ ] Interest and Disinterest Signalling for document routing: Create a `$blob_status` message to express `have`, `want` signalling. This can steer bundle creation and an eventual `--for` flag at bundle creation time to customize a bundle to a particular user.
- - [ ] Add a schema for `$peer_status`. Eg: `block`, `unblock`, `follow`, `unfollow`.
-
-# Idea Bin
-
- - [ ] Ability to add map/reduce plugins to support custom indices?
- - [ ] Ability to add a blob in one swoop using File objects and `Message#[]=`, maybe?
- - [ ] Bundling via [Optar](http://ronja.twibright.com/optar/) or [Colorsafe](https://github.com/colorsafe/colorsafe)
-
- - [ ] Ability to add map/reduce plugins to support custom indices?
- - [ ] Ability to add a blob in one swoop using File objects and `Message#[]=`, maybe?
- - [ ] Bundling via [Optar](http://ronja.twibright.com/optar/) or [Colorsafe](https://github.com/colorsafe/colorsafe)
+MOVED TO https://github.com/PigeonProtocolConsortium

bin/pigeon-cli

@@ -101,7 +101,7 @@ module Pigeon
     desc "append", "Add a key/value pair to the current DRAFT"
 
     def append(key, raw_value = "")
-      v = raw_value != "" ? raw_value : STDIN.read
+      v = raw_value != "" ? raw_value : STDIN.read.chomp
       if db.get_draft
         db.update_draft(key, v)
         puts db.get_draft.render_as_draft
@@ -191,10 +191,10 @@ module Pigeon
     desc "identity SUBCOMMAND ...ARGS", "Manage `.pgn` identity"
     subcommand "identity", Identity
 
-    desc "message SUBCOMMAND ...ARGS", "Manage blob storage"
+    desc "message SUBCOMMAND ...ARGS", "Manage text-based messages"
     subcommand "message", PigeonMessage
 
-    desc "peer SUBCOMMAND ...ARGS", "Manage blob storage"
+    desc "peer SUBCOMMAND ...ARGS", "Manage blob (file) storage"
     subcommand "peer", Peer
   end
 end

lib/pigeon.rb

@@ -17,7 +17,6 @@ module Pigeon
     "author <%= author %>",
     "depth <%= depth %>",
     "kind <%= kind %>",
-    "lipmaa <%= lipmaa %>",
     "prev <%= prev %>",
     "\n",
   ].join("\n")
@@ -55,7 +54,7 @@ module Pigeon
   STRING_KEYS_ONLY = "String keys only"
   MISSING_BODY = "BODY CANT BE EMPTY"
   STILL_HAVE_DRAFT = "RESET DRAFT (%s) FIRST (db.delete_current_draft)"
-  MISSING_DRAFT = "NO DRAFT. CREATE ONE FIRST. Call db.new_draft(kind, body)"
+  MISSING_DRAFT = "NO DRAFT. CREATE ONE FIRST. Call db.new_draft(kind:, body:)"
   RUNAWAY_LOOP = "RUNAWAY LOOP DETECTED"
 
   # Constants for internal use only:
@@ -91,40 +90,6 @@ module Pigeon
       "X" => 0b11101, "Y" => 0b11110, "Z" => 0b11111,
     }.freeze
 
-    def self.lipmaa(n)
-      # The original lipmaa function returns -1 for 0
-      # but that does not mesh well with our serialization
-      # scheme. Comments welcome on this one.
-      if n < 1 # Prevent -1, division by zero etc..
-        return nil
-      end
-
-      m, po3, x = 1, 3, n
-      # find k such that (3^k - 1)/2 >= n
-      while (m < n)
-        po3 *= 3
-        m = (po3 - 1) / 2
-      end
-      po3 /= 3
-      # find longest possible backjump
-      if (m != n)
-        while x != 0
-          m = (po3 - 1) / 2
-          po3 /= 3
-          x %= m
-        end
-        if (m != po3)
-          po3 = m
-        end
-      end
-      result = n - po3
-      if result == n - 1
-        return nil
-      else
-        return result
-      end
-    end
-
     # http://www.crockford.com/wrmg/base32.html
     def self.b32_encode(string)
       string
@@ -176,12 +141,6 @@ module Pigeon
       draft.author = author
       draft.depth = depth
       draft.prev = db.get_message_by_depth(mhash, depth - 1)
-      lpma = Helpers.lipmaa(depth)
-      if lpma && draft.prev
-        draft.lipmaa = db.get_message_by_depth(mhash, lpma)
-      else
-        draft.lipmaa = NOTHING
-      end
 
       unsigned = template.render_without_signature
       draft.signature = author.sign(unsigned)
@@ -214,13 +173,6 @@ module Pigeon
       count = db.get_message_count_for(author.multihash)
       expected_prev = db.get_message_by_depth(author.multihash, count - 1) || Pigeon::NOTHING
       assert("depth", count, msg.depth)
-      expected_lpma = Helpers.lipmaa(msg.depth)
-      if expected_lpma
-        real = db.get_message_by_depth(author.multihash, expected_lpma)
-        assert("lipmaa", msg.lipmaa, real)
-      else
-        assert("lipmaa", msg.lipmaa, NOTHING)
-      end
       assert("prev", msg.prev, expected_prev)
       tpl = msg.template.render_without_signature
       Helpers.verify_string(author, signature, tpl)
@@ -252,16 +204,7 @@ module Pigeon
 
     def self.hash2file_path(mhash)
       mhash = mhash.sub(BLOB_SIGIL, "")
-
-      [
-        mhash[0...7],
-        mhash[7...14],
-        mhash[14...21],
-        mhash[21...28],
-        mhash[28...35],
-        mhash[35...42],
-        [mhash[42...49], ".", mhash[49...52]].join(""),
-      ]
+      ["#{mhash[0..3]}#{mhash[-4..-1]}.blb"]
     end
 
     def self.decode_multihash(string)

lib/pigeon/database.rb

@@ -67,14 +67,12 @@ module Pigeon
                         body:,
                         depth:,
                         kind:,
-                        lipmaa:,
                         prev:,
                         signature:)
       msg = Message.new(author: RemoteIdentity.new(author),
                         kind: kind,
                         body: body,
                         prev: prev,
-                        lipmaa: lipmaa,
                         signature: signature,
                         depth: depth)
       _save_message(msg)
@@ -147,27 +145,34 @@ module Pigeon
         .join(BUNDLE_MESSAGE_SEPARATOR)
 
       File.write(File.join(file_path, MESSAGE_FILE), content + CR)
-    rescue => w
-      require "pry"
-      binding.pry
     end
 
     def import_bundle(file_path = DEFAULT_BUNDLE_PATH)
       bundle = File.read(File.join(file_path, MESSAGE_FILE))
       tokens = Pigeon::Lexer.tokenize(bundle)
       messages = Pigeon::Parser.parse(self, tokens)
-
+      wanted = Set.new
       messages
         .map(&:collect_blobs)
         .flatten
         .uniq
         .map do |mhash|
-        rel_path = Helpers.hash2file_path(mhash)
-        from = File.join([file_path] + rel_path)
-        to = File.join([DEFAULT_BLOB_DIR] + rel_path)
-        if (File.file?(from) && !File.file?(to))
-          data = File.read(from)
-          Helpers.write_to_disk(DEFAULT_BLOB_DIR, mhash, data)
+        b32 = mhash.gsub(BLOB_SIGIL, "")
+        binary = Pigeon::Helpers.b32_decode(b32)
+        wanted.add(binary)
+      end
+      all_files = Dir[File.join(file_path, "*.blb"), File.join(file_path, "*.BLB")]
+      all_files.map do |path|
+        data = File.read(path)
+        raw_digest = Digest::SHA256.digest(data)
+        if wanted.member?(raw_digest)
+          mhash = BLOB_SIGIL + Helpers.b32_encode(raw_digest)
+          rel_path = Helpers.hash2file_path(mhash)
+          from = File.join([file_path] + rel_path)
+          to = File.join([DEFAULT_BLOB_DIR] + rel_path)
+          if !File.file?(to)
+            Helpers.write_to_disk(DEFAULT_BLOB_DIR, mhash, data)
+          end
         end
       end
       messages

lib/pigeon/draft.rb

@@ -2,7 +2,7 @@ require "digest"
 
 module Pigeon
   class Draft
-    attr_accessor :signature, :prev, :lipmaa, :kind, :depth,
+    attr_accessor :signature, :prev, :kind, :depth,
                   :body, :author
 
     def initialize(kind:, body: {})
@@ -12,7 +12,6 @@ module Pigeon
       @depth = -1
       @body = {}
       @author = Pigeon::NOTHING
-      @lipmaa = Pigeon::NOTHING
       body.to_a.map { |(k, v)| self[k] = v }
     end
 

lib/pigeon/draft_serializer.rb

@@ -11,7 +11,6 @@ module Pigeon
       depth = DRAFT_PLACEHOLDER
       prev = DRAFT_PLACEHOLDER
       signature = DRAFT_PLACEHOLDER
-      lipmaa = DRAFT_PLACEHOLDER
       ERB.new([HEADER_TPL, BODY_TPL].join("")).result(binding)
     end
   end

lib/pigeon/lexer.rb

@@ -76,7 +76,6 @@ module Pigeon
     SEPERATOR = /\n/
     AUTHOR = /author #{FEED_VALUE}\n/
     DEPTH = /depth #{NUMERIC}\n/
-    LIPMAA = /lipmaa (#{MESG_VALUE}|#{NULL_VALUE})\n/
     PREV = /prev (#{MESG_VALUE}|#{NULL_VALUE})\n/
     KIND = /kind #{ALPHANUMERICISH}\n/
     BODY_ENTRY = /#{ALPHANUMERICISH}:#{ANY_VALUE}\n/
@@ -106,8 +105,7 @@ module Pigeon
       AUTHOR: [:FOOTER_SEPERATOR, :START],
       DEPTH: [:AUTHOR],
       KIND: [:DEPTH],
-      LIPMAA: [:KIND],
-      PREV: [:LIPMAA],
+      PREV: [:KIND],
       HEADER_SEPERATOR: [:PREV],
     }
 
@@ -136,13 +134,6 @@ module Pigeon
         return
       end
 
-      if scanner.scan(LIPMAA)
-        depth = scanner.matched.chomp.gsub("lipmaa ", "")
-        @tokens << [:LIPMAA, depth, scanner.pos]
-        check_header_order(:LIPMAA)
-        return
-      end
-
       if scanner.scan(PREV)
         prev = scanner.matched.chomp.gsub("prev ", "")
         @tokens << [:PREV, prev, scanner.pos]

lib/pigeon/message.rb

@@ -2,7 +2,7 @@ require "digest"
 
 module Pigeon
   class Message
-    attr_reader :author, :kind, :body, :signature, :depth, :lipmaa, :prev
+    attr_reader :author, :kind, :body, :signature, :depth, :prev
 
     def render
       template.render.chomp
@@ -20,7 +20,6 @@ module Pigeon
                    body:,
                    depth:,
                    prev:,
-                   lipmaa:,
                    signature:)
       raise MISSING_BODY if body.empty?
 
@@ -29,7 +28,6 @@ module Pigeon
       @depth = depth
       @kind = kind
       @prev = prev || Pigeon::NOTHING
-      @lipmaa = lipmaa
       @signature = signature
     end
 

lib/pigeon/message_serializer.rb

@@ -26,7 +26,6 @@ module Pigeon
       depth = message.depth
       kind = message.kind
       prev = message.prev || NOTHING
-      lipmaa = message.lipmaa || NOTHING
       signature = message.signature
 
       ERB.new(template).result(binding)

lib/pigeon/parser.rb

@@ -22,7 +22,6 @@ module Pigeon
         when :KIND then set(:kind, token[1])
         when :DEPTH then set(:depth, token[1])
         when :PREV then set(:prev, token[1])
-        when :LIPMAA then set(:lipmaa, token[1])
         when :HEADER_END then set(:body, {})
         when :BODY_ENTRY then set(token[1], token[2], @scratchpad[:body])
         when :BODY_END then nil

ruby_tutorial.md

@@ -12,13 +12,12 @@ This document will teach you how to:
  * Build messages using drafts.
  * Manage and query existing messages.
  * Replicate a database among peers.
- * Go beyond simple text messages and attach files to messages.
+ * Attach binary files to messages.
  * Communicate with remote databases using "bundles".
 
 This guide assumes you are familiar with Ruby and the Pigeon Protocol. For an introduction to the protocol, see our protocol specification [here](https://tildegit.org/PigeonProtocolConsortium/protocol_spec).
 
-Pigeon strive to have a "natural" API rather than a simple one. We will cover the API methods listed below. These are the only methods you will need to know to build a pigeon-based application:
-
+Below is a list of all methods needed to run a Pigeon node. Pigeon strives to have a _natural_ API rather than a simple one, which means you may not need to know every single method to operate a node successfully.
 
 **BLOB METHODS:** `#add_blob`,`#get_blob`
 
@@ -32,7 +31,7 @@ Pigeon strive to have a "natural" API rather than a simple one. We will cover th
 
 **PEER METHODS:** `#all_peers`, `#add_peer`, `#remove_peer`, `#all_blocks`, `#block_peer`, `#peer_blocked`
 
-Once you understand the methods listed above, you will have everything you need to start writing Pigeon-based applications. Please let us know what you build! Send an email to `contact` at `vaporsoft.xyz` with your progress.
+**Note to application developers:** Please let us know what you build! Send an email to `contact` at `vaporsoft.xyz` with your progress.
 
 ## Installation
 
@@ -40,7 +39,7 @@ Installation steps change over time. Please see [README.md](README.md) for the m
 
 ## Creating a Database Object
 
-When building Pigeon-based applications, a `Pigeon::Database` object controls nearly all interactions with the database.
+When building Pigeon-based applications, a `Pigeon::Database` object controls all interactions with the database.
 For the rest of the tutorial we will use the variable name `db` to refer to the current database.
 
 You can create your own database with the following steps:
@@ -73,67 +72,76 @@ As a convenience, the Pigeon Ruby client allows developers to keep zero or one "
 
 A draft is not part of the protocol spec. It is a convenience provided to users of this library. You could absolutely write messages by hand, calculate their signatures, convert everything to Base32 and manually add them to the database. This would be extremely tedious, however, so the draft functionality was added for convenience.
 
+The examples below will center around the creation of a fictitious gardening journal app, a use case that Pigeon is well suited for.
+
 Let's see if we have a draft to work with:
 
 ```ruby
 db.get_draft
-# => #<Pigeon::Draft:0x000056160b2e64a0 @author="NONE", @body={"a"=>"\"bar\"", "b"=>"&CH...QG.sha256"}, @depth=-1, @kind="unit_test", @lipmaa="NONE", @prev="NONE", @signature="NONE">
+RuntimeError: NO DRAFT. CREATE ONE FIRST. Call db.new_draft(kind:, body:)
+from lib/pigeon/database.rb:104:in `get_draft'
 ```
 
-It appears that my database has a draft. I don't actually remember what this draft was, so I will just delete it before proceeding.
+We do not have a draft yet. We need to create one:
 
 ```ruby
-db.delete_current_draft
-# => nil
+db.new_draft(kind: "garden_diary", body: {"message_text"=>"Tomato plant looking healthy."})
+=> #<Pigeon::Draft:0x00005603ed399b48 @author="NONE",
+#    @body={"greeting"=>"\"Hello, world!\""}, @depth=-1, @kind="example123",
+#    @prev="NONE", @signature="NONE">
 ```
 
-Now I can create a new draft. I am going to create a new `garden_diary` for a fictitious gardening app. In my gardening app, I expect every `garden_diary` message to have a `message_text` entry in its body. We can add that now.
+The command above creates a new draft entry of kind `garden_entry` with one key/value pair in the body. We can view the draft at any time via `#get_draft`:
 
 ```ruby
-db.new_draft(kind: "garden_diary", body: {"message_text" => "Tomato plant looking healthy."})
-# => #<Pigeon::Draft:0x000056160b63da68 @author="NONE", @body={"message_text"=>"\"Tomato plant looking healthy.\""}, @depth=-1, @kind="garden_diary", @lipmaa="NONE", @prev="NONE", @signature="NONE">
+db.get_draft
+# => #<Pigeon::Draft:0x00005603ed81e830 @author="NONE",
+#      @body={"greeting"=>"\"Hello, world!\""}, @depth=-1,
+#      @kind="example123", @prev="NONE",
+#      @signature="NONE">
 ```
 
-A few notes about this draft message:
-
- * `"garden_diary` is the message `kind`. This is definable by application developers and helps determine the type of message we are dealing with. A fictitious diary app might have other entries such as `"status_update"` or `"photo_entry"`. It depends on the application you are building.
- * Notice that my hash used string keys for the `"message_text"` body entry. You can only use strings for key / value pairs (no `:symbols` or numbers). Later on we will learn how to attach files to messages.
- * The `body:` part is optional. I could have called `db.new_draft(kind: "garden_diary")` and added key / value pairs to the body later.
-
-Oops! Speaking of adding entries to a draft's body, it looks like I forgot something. In my fictitious gardening app, a `garden_diary` entry doesn't just have a `"message_text"`, it also has a `"current_mood"` entry. Luckily, it is easy to add keys to unpublished drafts. Let's add the key now:
+Since the draft has not been published to the feed, its contents are mutable. We can add a new key/value pair to the message body with the following command:
 
 ```ruby
 db.update_draft("current_mood", "Feeling great")
 # => "\"Feeling great\""
 ```
 
-OK, I think our draft message is looking better. Let's take a look:
+A few notes about this draft message:
 
-```ruby
-db.get_draft
-# => => #<Pigeon::Draft:0x000056160b3e6be8 @author="NONE", @body={"message_text"=>"\"Tomato plant looking healthy.\"", "current_mood"=>"\"Feeling great\""}, @depth=-1, @kind="garden_diary", @lipmaa="NONE", @prev="NONE", @signature="NONE">
-```
+ * `"garden_diary` is the message `kind`. This is definable by application developers and helps determine the type of message we are dealing with. A fictitious diary app might have other entries such as `"status_update"` or `"photo_entry"`. It depends on the application you are building.
+ * I used string keys for the `"message_text"` body entry rather than symbols or numbers. This is because you can only use strings for key / value pairs (no `:symbols` or numbers). Later on we will learn how to attach files to messages.
+ * The `body:` part is optional. I could have called `db.new_draft(kind: "garden_diary")` and added key / value pairs to the body later.
 
-I can see the status of my current draft message using `db.get_draft`. It returns a `Pigeon::Draft` object, whish is not very human readable. To get a more human readable version, I can use the `render_as_draft` method on a `Draft` object:
+Let's take a final look at our draft message. To get a more human readable version, I can use the `render_as_draft` method on a `Draft` object:
 
 ```ruby
 human_readable_string = db.get_draft.render_as_draft
 # => "author DRAFT\nkind garden_dia...."
 puts human_readable_string
-# =>  author DRAFT
-#     kind garden_diary
-#     prev DRAFT
-#     depth DRAFT
-#     lipmaa DRAFT
+# => author DRAFT
+#    depth DRAFT
+#    kind garden_diary
+#    prev DRAFT
 #
-#     message_text:"Tomato plant looking healthy."
-#     current_mood:"Feeling great"
+#    greeting:"Hello, world!"
+#    current_mood:"Feeling great"
 ```
 
 Some interesting things about the draft we just rendered:
 
  * Unlike a message, a draft has no signature (yet).
- * The `author`, `kind`, `prev`, `depth`, `lipmaa` properties are all set to `"DRAFT"`. Real values will be populated when we finally publish the draft.
+ * The `author`, `kind`, `prev`, `depth` properties are all set to `"DRAFT"`. Real values will be populated when we finally publish the draft.
+
+If we want to start over, we can delete a draft via `delete_current_draft`:
+
+```ruby
+db.delete_current_draft
+# => nil
+```
+
+Let's not do this, though. Instead, we will publish this draft in the next section.
 
 ## Turning Drafts Into Messages
 
@@ -142,33 +150,31 @@ Since we did that in the last step, I will go ahead and publish the message:
 
 ```
 my_message = db.publish_draft
-# => #<Pigeon::Message:0x000056160b50dd00
-#  @author=#<Pigeon::RemoteIdentity:0x000056160b50dd78 @multihash="@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519">,
-#  @body={"message_text"=>"\"Tomato plant looking healthy.\"", "current_mood"=>"\"Feeling great\""},
-#  @depth=0,
-#  @kind="garden_diary",
-#  @lipmaa=0,
-#  @prev="NONE",
-#  @signature="2ZHC8TX3P2SQVQTMFYXTAT4S02RN43JNZNECRJDA7QMSJNE5G7NV7GTRK3PGFHFY9MBE1Q95BCKBSJH4V0PTX6945A34Z1CARTGH230.sig.ed25519">
+=> #<Pigeon::Message:0x000055a751032c28
+ @author=#<Pigeon::RemoteIdentity:0x000055a751032cf0 @multihash="USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG">,
+ @body={"greeting"=>"\"Hello, world!\"", "current_mood"=>"\"Feeling great\""},
+ @depth=0,
+ @kind="garden_diary",
+ @prev="NONE",
+ @signature="QNY...208">
 ```
 
 Let's look at our new message in a human-readable way:
 
 ```ruby
 puts my_message.render
-# =>  author @753...T6G.ed25519
-#     kind garden_diary
-#     prev NONE
-#     depth 0
-#     lipmaa 0
+# author USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG
+# depth 0
+# kind garden_diary
+# prev NONE
 #
-#     message_text:"Tomato plant looking healthy."
-#     current_mood:"Feeling great"
+# greeting:"Hello, world!"
+# current_mood:"Feeling great"
 #
-#     signature 2ZH...230.sig.ed25519
+# signature QNY...208
 ```
 
-We see that unlike our draft, the message has a signature. The header fields are also populated.
+Unlike our draft, the message has a signature. The header fields are also populated.
 
 In the next section, we will learn more about messages.
 
@@ -177,30 +183,28 @@ In the next section, we will learn more about messages.
 Drafts can be helpful when you are building a message incrementally and need a place to temporarily store things between application restarts.
 What about when you have all the information you need and want to publish immediately?
 
-In those cases, you can call `db.add_message` and your message will be published to your database immediately. No intermediate steps:
+In those cases, you can call `db.add_message` and your message will be published to your database immediately:
 
 ```ruby
 message = db.add_message("garden_entry", {"message_text" => "The basil is just OK", "current_mood" => "content"})
-# => #<Pigeon::Message:0x000056160b5cb558
-#  @author=#<Pigeon::RemoteIdentity:0x000056160b5cb5a8 @multihash="@753...T6G.ed25519">,
+# => #<Pigeon::Message:0x00005653352af998
+#  @author=#<Pigeon::RemoteIdentity:0x00005653352afa38 @multihash="USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG">,
 #  @body={"message_text"=>"\"The basil is just OK\"", "current_mood"=>"\"content\""},
 #  @depth=1,
 #  @kind="garden_entry",
-#  @lipmaa=0,
-#  @prev="%EM7...260.sha256",
-#  @signature="J59...238.sig.ed25519">
+#  @prev="TEXT.NPNQZAP9CB79GP8J0SN52F38EBJ9WV370HX6MVZD3XB804TVQQB0",
+#  @signature="95E...J3G">
 
 puts message.render
-# =>  author @753...T6G.ed25519
-#     kind garden_entry
-#     prev %EM7...260.sha256
-#     depth 1
-#     lipmaa 0
+# author USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG
+# depth 1
+# kind garden_entry
+# prev TEXT.NPNQZAP9CB79GP8J0SN52F38EBJ9WV370HX6MVZD3XB804TVQQB0
 #
-#     message_text:"The basil is just OK"
-#     current_mood:"content"
+# message_text:"The basil is just OK"
+# current_mood:"content"
 #
-#     signature J59...238.sig.ed25519
+# signature 95E...J3G
 ```
 
 We should now have 2 messages in the local database.
@@ -208,7 +212,7 @@ Let's take a look using the `db.all_messages` method:
 
 ```ruby
 db.all_messages
-# => ["%EM749647YHD3CBEC19TJJ7YME7BDXJ2KZ38ZZKS6E3VA0JHAM260.sha256", "%0HTM1H6ETBMKCPP5JMN2XEM060RYQHJ8P5KY09WRPTTVZ20N3EFG.sha256"]
+# => ["TEXT.NPN...QB0", "TEXT.444...92G"]
 ```
 
 The `#all_messages` method returns an array containing every message multihash in the database. We can then pass the multihash to the `db.read_message` method to retrieve the corresponding `Pigeon::Message` object.
@@ -216,107 +220,109 @@ The `#all_messages` method returns an array containing every message multihash i
 Let's look at the old log message we created from a draft previously:
 
 ```ruby
-old_message = db.read_message("%EM749647YHD3CBEC19TJJ7YME7BDXJ2KZ38ZZKS6E3VA0JHAM260.sha256")
-# => #<Pigeon::Message:0x000056160b35f580
-#  @author=#<Pigeon::RemoteIdentity:0x000056160b35ee50 @multihash="@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519">,
-#  @body={"message_text"=>"\"Tomato plant looking healthy.\"", "current_mood"=>"\"Feeling great\""},
-#  @depth=0,
-#  @kind="garden_diary",
-#  @lipmaa=0,
-#  @prev="NONE",
-#  @signature="2ZHC8TX3P2SQVQTMFYXTAT4S02RN43JNZNECRJDA7QMSJNE5G7NV7GTRK3PGFHFY9MBE1Q95BCKBSJH4V0PTX6945A34Z1CARTGH230.sig.ed25519">
+old_message = db.read_message("TEXT.444CC4NFHGQDQEZ6B6HSEPNZAZ80RSQF8TCAX8QR9NBR5T0XX92G")
+# => #<Pigeon::Message:0x0000565335384f08
+#  @author=#<Pigeon::RemoteIdentity:0x0000565335384da0 @multihash="USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG">,
+#  @body={"message_text"=>"\"The basil is just OK\"", "current_mood"=>"\"content\""},
+#  @depth=1,
+#  @kind="garden_entry",
+#  @prev="TEXT.NPNQZAP9CB79GP8J0SN52F38EBJ9WV370HX6MVZD3XB804TVQQB0",
+#  @signature="95E...J3G">
 
 puts old_message.render
-# author @753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519
-# kind garden_diary
-# prev NONE
-# depth 0
-# lipmaa 0
-#
-# message_text:"Tomato plant looking healthy."
-# current_mood:"Feeling great"
-#
-# signature 2ZHC8TX3P2SQVQTMFYXTAT4S02RN43JNZNECRJDA7QMSJNE5G7NV7GTRK3PGFHFY9MBE1Q95BCKBSJH4V0PTX6945A34Z1CARTGH230.sig.ed25519
+# author USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG
+# depth 1
+# kind garden_entry
+# prev TEXT.NPNQZAP9CB79GP8J0SN52F38EBJ9WV370HX6MVZD3XB804TVQQB0
+
+# message_text:"The basil is just OK"
+# current_mood:"content"
+
+# signature 95E...J3G
 ```
 
 Additionally, there is a `have_message?` helper that let's us know if we have a message in the local DB. It will return a `Pigeon::Message` (if found) or `false`:
 
-```
-db.have_message?("%AAAM1H6ETBBBCPP5JMN2XEM060RYQCCCP5KY09WRPTTVZ20N3FFF.sha256")
+```ruby
+db.have_message?("TEXT.QPNQGRBREXN4CB49RFZ8SQGXD98Z46FS08QH5ZATT6NE2HACC40X")
 # => false
 ```
 
 ## Working with Peers
 
-Building a gardening diary is not very fun unless there is a way of sharing your work. Pigeon supports data transfer through the use of peers.
+---
+STOPPED HERE.
 
-Every Pigeon database (including ours) has a unique identifier to identify itself.
+Building a gardening diary is not very fun unless there is a way of sharing your work. Pigeon supports data transfer through the use of peers. When a peer adds you to their local machine, they begin replicating your database on their machine, thereby giving them access to your diary entries and also creating a redundant backup.
 
-Let's call `db.who_am_i` to find out what our database multihash is:
+As we learned in the last section, Pigeon messages have a unique ID ("multihash"). Every message starts with the word "TEXT." and is followed by a base32 string of digits and letters. Much like messages, every Pigeon database has a unique identifier to identify itself. A database's multihash starts with the word "USER.". Like a message multihash, it is a long string of base32 characters:
+
+```
+USER.58844MCNB7ZF7HVKYFRBR7R7E75T8YXP4JBR267AS09RNMHEG3EG
+```
+
+We can call `db.who_am_i` to determine our local database multihash:
 
 ```ruby
 me = db.who_am_i
-# => #<Pigeon::LocalIdentity:0x000056160b5ca658
-#  @multihash="@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519",
-#  @seed="REDACTED",
-#  @signing_key=#<Ed25519::SigningKey:REDACTED>>
+# => #<Pigeon::LocalIdentity:0x0000558496ad4bf0 @seed="___", @signing_key=#<Ed25519::SigningKey:0x0000558496ad4bc8>>
 ```
 
-Calling `db.who_am_i` returned a `Pigeon::LocalIdentity`. To get results in a more copy/pastable format, call `#multihash` on the `LocalIdentity`:
+`db.who_am_i` returns a `Pigeon::LocalIdentity` rather than a string (which is what we want). To get results in a more copy/pastable format, call `#multihash` on the `LocalIdentity` object:
 
 ```ruby
-me.multihash
-# => "@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"
+puts me.multihash
+# USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG
 ```
 
 You can send this string to all your friends so they can add you as a peer to their respective databases.
 Let's add a friend to our database now.
-My friend has informed me her Pigeon identity is `"@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"`:
+My friend has informed me her Pigeon identity is `"USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"`:
 
 ```ruby
-db.add_peer("@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519")
-# => "@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"
+db.add_peer("USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G")
+# => "USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"
 ```
 
-My client will now keep a local copy of my friend's DB on disk at all times. Since Pigeon is an offline-only protocol, she will need to mail me an SD Card with her files. We will cover this later in the "Bundles" section.
+My client will now keep a local copy of my friend's DB on disk at all times, assuming I have received a "bundle" for the user. Since Pigeon is an offline-only protocol, she will need to mail me an SD Card with her files. We will cover this later in the "Bundles" section.
 
 If you ever lose track of who your peers are, you can call `db.all_peers` to get a list:
 
 ```ruby
 db.all_peers
-# => ["@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"]
+# => ["USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"]
 ```
 
 You can also remove peers if you no longer need to replicate their messages:
 
 ```ruby
-db.remove_peer("@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519")
-# => "@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"
+db.remove_peer("USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G")
+# => "USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"
 db.all_peers
 # => []
 ```
 
-It is also possible to block peers as needed via `db.block_peer`. `block_peer` is _not_ the same as `remove_peer`. Blocking a peer will prevent gossip from flowing through your database. All of their messages will be ignored and none of your peers will be able to retrieve their messages through you via gossip:
+It is also possible to block peers as needed via `db.block_peer`. `block_peer` is _not_ the same as `remove_peer`. Blocking a peer will prevent gossip from flowing through your database. All of their messages will be ignored and none of your peers will be able to retrieve their messages through you via gossip. When you block a peer, you are essentially imposing an embargo against them. Bundles from blocked peers will neither be imported nor exported under any circumstance.
 
 ```ruby
-db.block_peer("@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519")
-# => "@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"
+db.block_peer("USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G")
+# => "USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"
 
 db.all_blocks
-# => ["@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519"]
+# => ["USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G"]
 
-db.peer_blocked?("@753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G.ed25519")
+db.peer_blocked?("USER.753FT97S1FD3SRYPTVPQQ64F7HCEAZMWVBKG0C2MYMS5MJ3SBT6G")
 # => true
 ```
 
 ## Querying the Database
 
-The client offers some simple query capabilities and indexes. More will be added at a later date. Please email `contact` at `vaporsoft.xyz` if you are interested in helping.
+The client offers minimal query capabilities and indexes. More will be added at a later date. Please email `contact` at `vaporsoft.xyz` if you are interested in helping. Feature requests are welcome.
 
 ### Fetch a Message by Feed Identity + Message Depth
 
 ```ruby
-my_peer = "@MF312A76JV8S1XWCHV1XR6ANRDMPAT2G5K8PZTGKWV354PR82CD0.ed25519"
+my_peer = "USER.MF312A76JV8S1XWCHV1XR6ANRDMPAT2G5K8PZTGKWV354PR82CD0"
 db.get_message_by_depth(my_peer, 1)
 # => "%6JD96QB2EQ30EN3DMHH50NXMR0RZ2GMH43P2DZB3HN6PE6NFE9A0.sha256"
 ```
@@ -330,38 +336,41 @@ db.get_message_count_for(my_peer)
 
 ## Attaching Files to Messages
 
-Pigeon supports file attachments in the form of [blobs](https://en.wikipedia.org/wiki/Binary_large_object).
+There are limits to the usefulness (and size) of text content for a Pigeon message. When building a gardening diary app, a user will eventually want to attach garden photos to their entries.
 
-Once you have added a blob to your local database, it can be attached to messages using the special blob multihash string.
+Pigeon supports message file attachments in the form of [blobs](https://en.wikipedia.org/wiki/Binary_large_object).
+
+Once you have added a blob to your local database, it can be attached to messages using the special blob multihash string. Passing arbitrary binary data as a string via `db.add_blob(binary_data)` will do two things:
+
+1. Store the file in your local database (unless we already have a copy)
+2. Return a 57 character blob multihash, which can be used to reference the file inside of a message.
 
 ```ruby
 binary_data = File.read("kitty_cat.gif")
 db.add_blob(binary_data)
-# => "&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256"
+# => "FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG"
 ```
 
 Creating a blob returns a blob multihash (`&FV0...MRG.sha256`) which can be attached to a message in the form of keys or values:
 
 ```ruby
-the_blob_from_before = "&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256"
-msg = db.add_message("photo", {"my_cat_picture" => "&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256"})
+the_blob_from_before = "FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG"
+msg = db.add_message("photo", {"my_cat_picture" => "FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG"})
 puts msg.render
-# => author @MF312A76JV8S1XWCHV1XR6ANRDMPAT2G5K8PZTGKWV354PR82CD0.ed25519
-#    kind photo
-#    prev %ZV85NQS8B1BWQN7YAME1GB0G6XS2AVN610RQTME507DN5ASP2S6G.sha256
-#    depth 3
-#    lipmaa 2
+# author USER.6DQ4RRNBKJ2T4EY5E1GZYYX6X6SZXV1W0GNH1HA4KGKA5KZ2Y2DG
+# depth 2
+# kind photo
+# prev TEXT.444CC4NFHGQDQEZ6B6HSEPNZAZ80RSQF8TCAX8QR9NBR5T0XX92G
 #
-#    my_cat_picture:&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256
+# my_cat_picture:FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG
 #
-#    signature JSPJJQJRVBVGV52K2058AR2KFQCWSZ8M8W6Q6PB93R2T3SJ031AYX1X74KCW06HHVQ9Y6NDATGE6NH3W59QY35M58YDQC5WEA1ASW08.sig.ed25519
-
+# signature WT96XNJ6T006YS51ZDVPT1A7DJW2E4BTBZF66WHHWMKEP35MZ0YD30C32M8WQ85VK19SQFK47MXPEDMWW1GC0RV5XPYHT6WNDMGZM1R
 ```
 
 If you want to retrieve a blob later, you can pass the blob multihash to `db#get_blob`. The client will return it as binary data.
 
 ```ruby
-db.get_blob("&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256")
+db.get_blob("FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG")
 # => "GIF89aX\u0000\u001F\u0000\xD58\u0000\u0000\u0000\u0000...
 ```
 
@@ -369,9 +378,12 @@ db.get_blob("&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256")
 
 Eventually, you will want to share your log messages with a peer, either as a form of communication or for the sake of creating redundant backups.
 
-All data transfer operations in Pigeon are file based. To export data from your local database, one must create a "bundle", which is a file directory with a very specific layout. Think of bundles as a specialized archive format that a Pigeon-compliant database can easily ingest. The bundle mechanism will **package all blobs and messages into a single exportable directory structure automatically**. As long as your peer's client is compliant with the Pigeon spec, they will be replicated onto the peer's machine upon import.
+All data transfer operations in Pigeon are file based. To export data from your local database, one must create a "bundle", which is a file directory with a very specific layout. You can take this directory structure and burn it to a DVD-R, Zip and email, host on an HTTP server, seed it as a torrent, etc..
 
-Pigeon does not specify transport or compression concerns, but any reliable file transfer method is possible.
+Think of bundles as a specialized archive format that a Pigeon-compliant database can easily ingest. The bundle mechanism will **package all blobs and messages into a single exportable directory structure automatically**. As long as your peer's client is compliant with the Pigeon spec, they will be replicated onto the peer's machine upon import.
+
+
+Pigeon does not specify transport or compression concerns, but any reliable file transfer method is possible. For example, applying GZip compression to a bundle is a great idea, but the spec itself does not dictate how to do this. This is particularly true for network-related concerns. The Pigeon spec will never reference network transport in the spec.
 
 In the example below, I will create a bundle called `"bundle_for_my_peer"`.
 
@@ -393,3 +405,13 @@ If you wish to ingest a peer's message, you can perform the operation in reverse
 db.import_bundle("a_bundle_my_peer_gave_me")
 ```
 
+# Wrapping Up
+
+That's all there is to the protocol spec. In summary:
+
+ * Messages are the core building block of Pigeon databases.
+ * Messages can have file attachments via "blobs"
+ * Messages can reference other users via "USER.****" multihashes
+ * Messages are replicated onto peer machines via "following"
+ * Messages can be avoided via "blocking"
+ * Messages are shared between machines by passing around a disk directory structure known as a "bundle".

spec/fixtures/has_blobs/messages.pgn

@@ -1,29 +1,26 @@
-author USER.4CZHSZAH8473YPHP1F1DR5ZVCRKEA4Q0BY18NMXYE14NZ0XV2PGG
+author USER.59X51RSZQZR15BX86VDWG37AAMVP43PTBWD1WS66FQFCDPHAQDZ0
 depth 0
 kind example
-lipmaa NONE
 prev NONE
 
 file_name:FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG
 
-signature H4SV73PSNKYTJZEA2ESBNDMV4D87K36T72E8FTTG1H74RXAMWJ6PVMQ421W2K6NXPAH5YS6B1PJCG2DVTEPWKBPTMY5T9ZZBXYX3020
+signature N33N7D8KFFVVPHTDE17JS7708YPAVF2F0F0AZS1FFW3D15ZH1K3HEFNQJK7KT7NMSAF8PDC1YDD5M57NPG2PTEEYPBKC1G3HFHN3J08
 
-author USER.4CZHSZAH8473YPHP1F1DR5ZVCRKEA4Q0BY18NMXYE14NZ0XV2PGG
+author USER.59X51RSZQZR15BX86VDWG37AAMVP43PTBWD1WS66FQFCDPHAQDZ0
 depth 1
 kind example
-lipmaa NONE
-prev TEXT.E90DY6RABDQ2CJPVQHYQDYH6N7Q46SZKQ0AQ76J6D684HYBRKE4G
+prev TEXT.S5G187G11N2T76E2TSPS40K5QEY6S9ZC68TKEVH7JBPN27VDTKY0
 
 file_name:FILE.YPF11E5N9JFVB6KB1N1WDVVT9DXMCHE0XJWBZHT2CQ29S5SEPCSG
 
-signature T2M98QY1P1FRYT4KRMRT1X5RQRY56HKTKPZEJDD5Y7W7HR57XE0RD5X4HF9YSTS9CBH4ZCJ4XM4NAY3SRFEFM6EY1RTV7HSE43A4P20
+signature 53454CZKNSBK4D8NZCKWRWWE37DVANJWCS891XGRR2M8M4AJP2XNTC86MQAWAMYX3W517KWW6JD9MX3FMXNNBQ1TJS5HSK9CTW9G018
 
-author USER.4CZHSZAH8473YPHP1F1DR5ZVCRKEA4Q0BY18NMXYE14NZ0XV2PGG
+author USER.59X51RSZQZR15BX86VDWG37AAMVP43PTBWD1WS66FQFCDPHAQDZ0
 depth 2
 kind example
-lipmaa NONE
-prev TEXT.7ZKXANAAM31R9AMHMBVGP9Q5BF5HSCP557981VQHBTRYETGTGAK0
+prev TEXT.5BBGSKGBHKYE6R0SJSZAGNEQA8PGJ5CMTQD1XGKKP2CHYPZR8G90
 
 file_name:FILE.622PRNJ7C0S05XR2AHDPKWMG051B1QW5SXMN2RQHF2AND6J8VGPG
 
-signature KFMGFGSCZ36J1FKM5J68SVJ7Y074CQR7PF73690ZN4PPRTYZNS28D76AFBYXX9N2F4Z13KKFNG3308ZTGPB13D5N5CBGGZBN4V8A210
+signature JVN1YPVA637NF6GGPCX8GXT5FXTZPA1YM68ZWQQNXYD36CX0PSDBHXQMY7PMJYMCPFYW5BR56P2GVETM8AVYSKAFSYPVM3F7KVDW020

spec/fixtures/normal/messages.pgn

@@ -1,29 +1,35 @@
-author USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0
+author USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0
 depth 0
-kind unit_test1
-lipmaa NONE
+kind nonsense
 prev NONE
 
+example:"Just block me"
+
+signature 0N0B419YSCHYM82YWGBB6VF0MHHCS0ACGBKD8MYMTGS59XC1T60W2JHKHEW9ZQJW53KTJMVB3MGV3JTFKZWQH9QMAAWG3DE6AQ6SJ30
+
+author USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0
+depth 1
+kind unit_test1
+prev TEXT.839FP9NB9E1KFG17SZF49X57B9AYNNX1HQ8DGVE940BPWCRXS82G
+
 foo:"bar"
 
-signature 2VMAG4SCX5RHVBKCB1RNZCB0AJN4WN6FEMS7W9FM1CVYSZXMX7CPQFCDPYEKCTGG91Y1YSGY4G5K8XAGQ67HEPDFRMRYQHWQBATAC2R
+signature 7YC6P4AJMPV3JH57JV0AHDP0ZV59WZYKHF49DEM2CJP5ZQCVR4XN6RMS18SBE5S2YXAFG05FA8S3B2YC35CH464822ZQXTCMN2F9G3R
 
-author USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0
-depth 1
+author USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0
+depth 2
 kind unit_test2
-lipmaa NONE
-prev TEXT.6CBA4J3756A5SNM1W1GHNCTT9EG95ZP3ZMAT5Z1EJP7TXMNNVZC0
+prev TEXT.XHHQMFDK1DQSVXQ0XJQDSZQWXF8BNQ1QNRZW4K9V34264MF3WSFG
 
 bar:"baz"
 
-signature Y34Q47V0BY370RM5KWGRJRN9HFNGJN0C3DEYVB2V2476CW9RN5HD4XD7KMQ6T4T42N36R5P3XX6E3FYEWVZR25AVCF6KQPZHJP6EM10
+signature 5YBYC1RSB27WZ00H567RP1YAYBW30PAVHG3ZG55VY2R137YMPZZ0ZMD4T7MJ8RYCMTT72AN4WCN7QAS1NPAPQE134TE8CX7PH2TFM2R
 
-author USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0
-depth 2
+author USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0
+depth 3
 kind unit_test3
-lipmaa NONE
-prev TEXT.5BQZVA8JDC77AVGMF45CMPVHRNXFHQ2C01QJEAR57N6K12JN6PAG
+prev TEXT.DG0BZ241KY8E60C1F88MTZNEDDBQFZS4EMCNR23VD6Y6RZGNEXSG
 
 cats:"meow"
 
-signature W68NWDQB2WTZ8T1RHP5BZA4N1STVKV16K0PXH10MZVR3XTF8HC7T8646X7SAKP5DFZ5K74QEKE3T2K6V0EST50YQQD7FD2PT0H8J62G
+signature QWHA8KHSVFBC0X84VH2F2BS3CSCY58ER4ETXH1WB8SEEMDBS0TBAQHA2HNK1W7VDATBVZHB7EHWNYEN86HYBKK7BBMNSSMR45CEG838

spec/fixtures/x/messages.pgn

@@ -1,9 +1,8 @@
-author USER.58844MCNB7ZF7HVKYFRBR7R7E75T8YXP4JBR267AS09RNMHEG3EG
+author USER.YJTH2BBAAAXK2RYKWRXYE0E0ANME1YPZPD8TV5VCS40X3D75AJ3G
 depth 0
 kind nonsense
-lipmaa NONE
 prev NONE
 
 example:"Just block me"
 
-signature 689FV4JQGS58PBXM5N69X1G3KSV4742B6H974GQ2Y1NW9EV7SA3GJPVVDH8MDQGNDF8QWM2AZDPJYRBVXFCRM6S2EE4Z1KMAXBH5018
+signature 3WD5EJCMZZ27JJ8EQZ3M5XNK3QZ40A5T894Q27ZVMBT4T65KVZWK5PC74F54DX5EB3D3EFYQJQ5C1D9X2J22QKAJHTSG3B7SC932W28

spec/pigeon/bundle_spec.rb

@@ -40,7 +40,8 @@ RSpec.describe Pigeon::Message do
 
   it "does not ingest messages from blocked peers" do
     db.reset_database
-    antagonist = "USER.58844MCNB7ZF7HVKYFRBR7R7E75T8YXP4JBR267AS09RNMHEG3EG"
+    expect(db.all_messages.count).to eq(0)
+    antagonist = "USER.YJTH2BBAAAXK2RYKWRXYE0E0ANME1YPZPD8TV5VCS40X3D75AJ3G"
     db.block_peer(antagonist)
     db.import_bundle(BLOCKED_PEER_FIXTURE_PATH)
     expect(db.all_messages.count).to eq(0)

spec/pigeon/draft_spec.rb

@@ -20,7 +20,6 @@ RSpec.describe Pigeon::Draft do
     "author DRAFT",
     "depth DRAFT",
     "kind unit_test",
-    "lipmaa DRAFT",
     "prev DRAFT",
     "\na:\"bar\"",
     "b:FILE.CHHABX8Q9D9Q0BY2BBZ6FA7SMAFNE9GGMSDTZVZZC9TK2N9F15QG",

spec/pigeon/helpers_spec.rb

@@ -1,22 +1,4 @@
 RSpec.describe Pigeon::Helpers do
-  it "creates lipmalinks" do
-    [
-      [-1, nil],
-      [0, nil],
-      [1, nil],
-      [2, nil],
-      [3, nil],
-      [4, 1],
-      [5, nil],
-      [6, nil],
-      [7, nil],
-      [8, 4],
-      [13, 4],
-    ].each do |(input, expected)|
-      expect(Pigeon::Helpers.lipmaa(input)).to eq(expected)
-    end
-  end
-
   it "handles Crockford Base 32 values" do
     10.times do
       raw_bytes = SecureRandom.random_bytes(32)

spec/pigeon/lexer_spec.rb

@@ -2,49 +2,48 @@ require "spec_helper"
 
 RSpec.describe Pigeon::Lexer do
   EXPECTED_TOKENS1 = [
-    [:AUTHOR, "USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0", 65],
+    [:AUTHOR, "USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0", 65],
     [:DEPTH, 0, 73],
-    [:KIND, "unit_test1", 89],
-    [:LIPMAA, "NONE", 101],
-    [:PREV, "NONE", 111],
-    [:HEADER_END, 112],
-    [:BODY_ENTRY, "foo", "\"bar\"", 122],
+    [:KIND, "nonsense", 87],
+    [:PREV, "NONE", 97],
+    [:HEADER_END, 98],
+    [:BODY_ENTRY, "example", "\"Just block me\"", 122],
     [:BODY_END, 123],
-    [:SIGNATURE,
-     "2VMAG4SCX5RHVBKCB1RNZCB0AJN4WN6FEMS7W9FM1CVYSZXMX7CPQFCDPYEKCTGG91Y1YSGY4G5K8XAGQ67HEPDFRMRYQHWQBATAC2R",
-     237],
+    [:SIGNATURE, "0N0B419YSCHYM82YWGBB6VF0MHHCS0ACGBKD8MYMTGS59XC1T60W2JHKHEW9ZQJW53KTJMVB3MGV3JTFKZWQH9QMAAWG3DE6AQ6SJ30", 237],
     [:MESSAGE_DELIM, 238],
-    [:AUTHOR, "USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0", 303],
+    [:AUTHOR, "USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0", 303],
     [:DEPTH, 1, 311],
-    [:KIND, "unit_test2", 327],
-    [:LIPMAA, "NONE", 339],
-    [:PREV, "TEXT.6CBA4J3756A5SNM1W1GHNCTT9EG95ZP3ZMAT5Z1EJP7TXMNNVZC0", 402],
-    [:HEADER_END, 403],
-    [:BODY_ENTRY, "bar", "\"baz\"", 413],
-    [:BODY_END, 414],
-    [:SIGNATURE,
-     "Y34Q47V0BY370RM5KWGRJRN9HFNGJN0C3DEYVB2V2476CW9RN5HD4XD7KMQ6T4T42N36R5P3XX6E3FYEWVZR25AVCF6KQPZHJP6EM10",
-     528],
-    [:MESSAGE_DELIM, 529],
-    [:AUTHOR, "USER.R68Q26P1GEFC0SNVVQ9S29SWCVVRGCYRV7D96GAN3XVQE3F9AZJ0", 594],
-    [:DEPTH, 2, 602],
-    [:KIND, "unit_test3", 618],
-    [:LIPMAA, "NONE", 630],
-    [:PREV, "TEXT.5BQZVA8JDC77AVGMF45CMPVHRNXFHQ2C01QJEAR57N6K12JN6PAG", 693],
-    [:HEADER_END, 694],
-    [:BODY_ENTRY, "cats", "\"meow\"", 706],
-    [:BODY_END, 707],
-    [:SIGNATURE,
-     "W68NWDQB2WTZ8T1RHP5BZA4N1STVKV16K0PXH10MZVR3XTF8HC7T8646X7SAKP5DFZ5K74QEKE3T2K6V0EST50YQQD7FD2PT0H8J62G",
-     821],
-    [:MESSAGE_DELIM, 821],
-  ].freeze
+    [:KIND, "unit_test1", 327],
+    [:PREV, "TEXT.839FP9NB9E1KFG17SZF49X57B9AYNNX1HQ8DGVE940BPWCRXS82G", 390],
+    [:HEADER_END, 391],
+    [:BODY_ENTRY, "foo", "\"bar\"", 401],
+    [:BODY_END, 402],
+    [:SIGNATURE, "7YC6P4AJMPV3JH57JV0AHDP0ZV59WZYKHF49DEM2CJP5ZQCVR4XN6RMS18SBE5S2YXAFG05FA8S3B2YC35CH464822ZQXTCMN2F9G3R", 516],
+    [:MESSAGE_DELIM, 517],
+    [:AUTHOR, "USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0", 582],
+    [:DEPTH, 2, 590],
+    [:KIND, "unit_test2", 606],
+    [:PREV, "TEXT.XHHQMFDK1DQSVXQ0XJQDSZQWXF8BNQ1QNRZW4K9V34264MF3WSFG", 669],
+    [:HEADER_END, 670],
+    [:BODY_ENTRY, "bar", "\"baz\"", 680],
+    [:BODY_END, 681],
+    [:SIGNATURE, "5YBYC1RSB27WZ00H567RP1YAYBW30PAVHG3ZG55VY2R137YMPZZ0ZMD4T7MJ8RYCMTT72AN4WCN7QAS1NPAPQE134TE8CX7PH2TFM2R", 795],
+    [:MESSAGE_DELIM, 796],
+    [:AUTHOR, "USER.3VX92CSQKDK854SYDMESAP6SQKKDMB5Q6XP6HVNETYS064BA0WP0", 861],
+    [:DEPTH, 3, 869],
+    [:KIND, "unit_test3", 885],
+    [:PREV, "TEXT.DG0BZ241KY8E60C1F88MTZNEDDBQFZS4EMCNR23VD6Y6RZGNEXSG", 948],
+    [:HEADER_END, 949],
+    [:BODY_ENTRY, "cats", "\"meow\"", 961],
+    [:BODY_END, 962],
+    [:SIGNATURE, "QWHA8KHSVFBC0X84VH2F2BS3CSCY58ER4ETXH1WB8SEEMDBS0TBAQHA2HNK1W7VDATBVZHB7EHWNYEN86HYBKK7BBMNSSMR45CEG838", 1076],
+    [:MESSAGE_DELIM, 1076],
+  ]
 
   MESSAGE_LINES = [
     "author @VG44QCHKA38E7754RQ5DAFBMMD2CCZQRZ8BR2J4MRHHGVTHGW670",
     "depth 0",
     "kind unit_test",
-    "lipmaa NONE",
     "prev NONE",
     "",
     "foo:\"bar\"",
@@ -94,25 +93,4 @@ RSpec.describe Pigeon::Lexer do
     expect(hash[:PREV]).to eq Pigeon::NOTHING
     expect(hash[:SIGNATURE]).to eq(message.signature)
   end
-
-  # it "catches syntax errors" do
-  #   e = Pigeon::Lexer::LexError
-  #   err_map = {
-  #     0 => "Syntax error pos 0 by START field in HEADER",
-  #     1 => "Syntax error pos 69 by AUTHOR field in HEADER",
-  #     2 => "Syntax error pos 77 by DEPTH field in HEADER",
-  #     3 => "Syntax error pos 92 by KIND field in HEADER",
-  #     4 => "Syntax error pos 104 by LIPMAA field in HEADER",
-  #     5 => "Syntax error pos 114 by PREV field in HEADER",
-  #     6 => "Syntax error pos 115 by HEADER_SEPERATOR field in BODY",
-  #     7 => "Syntax error pos 125 by A_BODY_ENTRY field in BODY",
-  #     8 => "Parse error at 126. Double carriage return not found.",
-  #   }
-  #   (0..8).to_a.map do |n|
-  #     t = MESSAGE_LINES.dup.insert(n, "TEXT.@@").join("\n")
-  #     emsg = err_map.fetch(n)
-  #     puts "=== #{n}:"
-  #     expect { Pigeon::Lexer.tokenize(t) }.to raise_error(e, emsg)
-  #   end
-  # end
 end

spec/pigeon/message_spec.rb

@@ -48,7 +48,6 @@ RSpec.describe Pigeon::Message do
       "author __AUTHOR__",
       "depth 0",
       "kind unit_test",
-      "lipmaa NONE",
       "prev NONE",
       "",
       "a:\"bar\"",
@@ -135,7 +134,6 @@ RSpec.describe Pigeon::Message do
       [:KIND, "invalid"],
       [:PREV, "NONE"],
       [:DEPTH, 10],
-      [:LIPMAA, "TEXT.4PE7S4XCCAYPQ42S98K730CEW6ME5HRWJKHHEGYVYPFHSJWXEY1G"],
       [:HEADER_END],
       [:BODY_ENTRY, "duplicate", "This key is a duplicate."],
       [:SIGNATURE, "DN7yPTE-m433ND3jBL4oM23XGxBKafjq0Dp9ArBQa_TIGU7DmCxTumieuPBN-NKxlx_0N7-c5zjLb5XXVHYPCQ=="],

spec/pigeon/parser_spec.rb

@@ -14,7 +14,6 @@ RSpec.describe Pigeon::Lexer do
     [:KIND, "invalid"],
     [:PREV, "NONE"],
     [:DEPTH, 0],
-    [:LIPMAA, Pigeon::Helpers.lipmaa(0)],
     [:HEADER_END],
     [:BODY_ENTRY, "duplicate", "Pigeon does not allow duplicate keys."],
     [:BODY_ENTRY, "duplicate", "This key is a duplicate."],
@@ -24,14 +23,22 @@ RSpec.describe Pigeon::Lexer do
 
   it "parses tokens" do
     results = Pigeon::Parser.parse(db, tokens)
-    expect(results.length).to eq(3)
+    expect(results.length).to eq(4)
+    expected_sigs = [
+      "0N0B419YSCHYM82YWGBB6VF0MHHCS0ACGBKD8MYMTGS59XC1T60W2JHKHEW9ZQJW53KTJMVB3MGV3JTFKZWQH9QMAAWG3DE6AQ6SJ30",
+      "7YC6P4AJMPV3JH57JV0AHDP0ZV59WZYKHF49DEM2CJP5ZQCVR4XN6RMS18SBE5S2YXAFG05FA8S3B2YC35CH464822ZQXTCMN2F9G3R",
+      "5YBYC1RSB27WZ00H567RP1YAYBW30PAVHG3ZG55VY2R137YMPZZ0ZMD4T7MJ8RYCMTT72AN4WCN7QAS1NPAPQE134TE8CX7PH2TFM2R",
+      "QWHA8KHSVFBC0X84VH2F2BS3CSCY58ER4ETXH1WB8SEEMDBS0TBAQHA2HNK1W7VDATBVZHB7EHWNYEN86HYBKK7BBMNSSMR45CEG838",
+    ].sort
+    actual_sigs = results.map { |x| x.signature }.sort
+    expect(actual_sigs - expected_sigs).to eq([])
     expect(results.first).to be_kind_of(Pigeon::Message)
     expect(results.last).to be_kind_of(Pigeon::Message)
   end
 
   it "ingests and reconstructs a bundle" do
     messages = db.import_bundle("./spec/fixtures/normal")
-    expect(messages.length).to eq(3)
+    expect(messages.length).to eq(4)
     expect(messages.map(&:class).uniq).to eq([Pigeon::Message])
     re_bundled = messages.map(&:render).join("\n\n") + "\n"
     expect(re_bundled).to eq(example_bundle)

spec/pigeon/template_spec.rb

@@ -1,13 +1,12 @@
 require "spec_helper"
 
 RSpec.describe Pigeon::MessageSerializer do
-  SHIM_ATTRS = %i[author body kind depth prev signature lipmaa].freeze
+  SHIM_ATTRS = %i[author body kind depth prev signature].freeze
   MessageShim = Struct.new(*SHIM_ATTRS)
   TOP_HALF = [
     "author FAKE_AUTHOR",
     "\ndepth 23",
     "\nkind FAKE_KIND",
-    "\nlipmaa 22",
     "\nprev NONE",
     "\n\nfoo:\"bar\"\n\n",
   ].join("")
@@ -28,7 +27,6 @@ RSpec.describe Pigeon::MessageSerializer do
       depth: 23,
       prev: nil,
       signature: "XYZ",
-      lipmaa: 22,
     }.values
     message = MessageShim.new(*params)
     template = Pigeon::MessageSerializer.new(message)