README Updates, add db#have_blob? helper

README.md

@@ -49,6 +49,10 @@ See `kitchen_sink.sh` examples.
  - [X] Create a contact email for project outsiders (and maybe a developer email list?)
  - [X] Update README.md
  - [X] Update Ruby API docs
+ - [ ] Make location of blob folder configurable
+ - [ ] Update blobs spec to clear out blob folder every time it runs.
+ - [ ] Filter messages from blocked users importing bundles.
+ - [ ] Filter blobs from blocked users when importing bundles.
  - [ ] Oops, `lipmaa` field needs to be a hash, not an integer!
  - [ ] Update Dev docs in protocol spec to reflect changes to `lipmaa` header.
  - [ ] Update spec document CLI usage examples to reflect API changes in 2020.

lib/pigeon/database.rb

@@ -151,12 +151,14 @@ module Pigeon
     def import_bundle(file_path = DEFAULT_BUNDLE_PATH)
       bundle = File.read(File.join(file_path, "messages.pgn"))
       tokens = Pigeon::Lexer.tokenize(bundle)
-      blobs = tokens.each_with_object(Set.new) do |(_a, b, c), set|
-        [b, c].map do |d|
-          set.add(d) if Helpers.blob_multihash?(d)
-        end
-      end
-      Pigeon::Parser.parse(self, tokens)
+      messages = Pigeon::Parser.parse(self, tokens)
+      messages
+        .map(&:collect_blobs)
+        .flatten
+        .uniq
+        .reject { |x| have_blob?(x) }
+        .map { |x| binding.pry }
+      messages
     end
 
     # === BLOBS
@@ -168,6 +170,10 @@ module Pigeon
       store.add_blob(b)
     end
 
+    def have_blob?(b)
+      store.have_blob?(b)
+    end
+
     # === DB Management
     def _get_config(k)
       store._get_config(k)

lib/pigeon/storage.rb

@@ -66,9 +66,9 @@ module Pigeon
     end
 
     def get_blob(blob_multihash)
-      path = File.join(Helpers.hash2file_path(blob_multihash))
-      path = File.join(PIGEON_BLOB_PATH, path)
-      File.read(path) if File.file?(path)
+      path1 = File.join(Helpers.hash2file_path(blob_multihash))
+      path2 = File.join(PIGEON_BLOB_PATH, path1)
+      File.read(path2) if File.file?(path2)
     end
 
     # `nil` means "none"

spec/pigeon/bundle_spec.rb

@@ -46,12 +46,16 @@ RSpec.describe Pigeon::Message do
   end
 
   it "ingests a bundle's blobs" do
+    puts "WARNING: This test deletes the blob dir! Fix ASAP"
+    puts "WARNING: This bundle contains blobs, but does not reference them."
+    `rm -rf #{Pigeon::PIGEON_BLOB_PATH}`
     db.reset_database
     db.import_bundle("./spec/fixtures/has_blobs")
     expect(db.all_messages.count).to eq(1)
     ["&622PRNJ7C0S05XR2AHDPKWMG051B1QW5SXMN2RQHF2AND6J8VGPG.sha256",
      "&FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG.sha256",
      "&YPF11E5N9JFVB6KB1N1WDVVT9DXMCHE0XJWBZHT2CQ29S5SEPCSG.sha256"].map do |h|
+      expect(db.have_blob?(h)).to be true
       expect(db.get_blob(h)).to be_kind_of(String)
     end
   end