diff --git a/Gemfile b/Gemfile index cdc6926..403535b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,3 @@ -# frozen_string_literal: true - source "https://rubygems.org" gem "ed25519" diff --git a/Gemfile.lock b/Gemfile.lock index b4c94af..65377d2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,4 +41,4 @@ DEPENDENCIES thor BUNDLED WITH - 2.0.2 + 2.1.4 diff --git a/README.md b/README.md index dc0e05e..e633cef 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,8 @@ TODO - [X] Check block list before ingesting bundles. - [X] Need a way of importing / exporting a feeds blobs. (see "Bundle Brainstorming" below) - [X] Need a way of adding peers messages / gossip to bundles. (see "Bundle Brainstorming" below) - - [ ] Rename `who_am_i` as `get_who_am_i` to follow VERB + NOUN convention. + - [X] Rename `who_am_i` as `get_who_am_i` to follow VERB + NOUN convention. + - [ ] Find that non-deterministic runaway loop in the test suite. - [ ] Update README.md / tutorial.rb (user manual for `Pigeon::Database`). - [ ] Update spec document CLI usage examples to reflect API changes in 2020. - [ ] Publish to RubyGems diff --git a/bin/pigeon-cli b/bin/pigeon-cli index 2e0edcf..8a37e57 100755 --- a/bin/pigeon-cli +++ b/bin/pigeon-cli @@ -1,5 +1,4 @@ #!/usr/bin/env ruby - require_relative "../lib/pigeon" require "thor" @@ -7,13 +6,13 @@ def db if File.file?(Pigeon::PIGEON_DB_PATH) $db ||= Pigeon::Database.new else - STDERR.puts("You must first run `pigeon-cli identity new`.") + warn("You must first run `pigeon-cli identity new`.") exit 1 end end def bail(msg) - $stderr.puts msg + warn msg exit 1 end @@ -55,7 +54,7 @@ module Pigeon desc "set", "Copy arbitrary binary data into the database" def set(data = "") - blob = (data != "") ? data : STDIN.read + blob = data != "" ? data : STDIN.read puts db.add_blob(blob) end @@ -102,7 +101,7 @@ module Pigeon desc "append", "Add a key/value pair to the current DRAFT" def append(key, raw_value = "") - v = (raw_value != "") ? raw_value : STDIN.read + v = raw_value != "" ? raw_value : STDIN.read if db.get_draft db.update_draft(key, v) puts db.get_draft.render_as_draft diff --git a/lib/pigeon/database.rb b/lib/pigeon/database.rb index 31d2d12..26b9ac8 100644 --- a/lib/pigeon/database.rb +++ b/lib/pigeon/database.rb @@ -8,22 +8,46 @@ module Pigeon end # === PEERS - def add_peer(p); store.add_peer(p); end - def block_peer(p); store.block_peer(p); end - def remove_peer(p); store.remove_peer(p); end - def peer_blocked?(p); store.peer_blocked?(p); end - def all_blocks(); store.all_blocks(); end - def all_peers(); store.all_peers(); end + def add_peer(p) + store.add_peer(p) + end + + def block_peer(p) + store.block_peer(p) + end + + def remove_peer(p) + store.remove_peer(p) + end + + def peer_blocked?(p) + store.peer_blocked?(p) + end + + def all_blocks + store.all_blocks + end + + def all_peers + store.all_peers + end # === MESSAGES - def all_messages(mhash = nil); store.all_messages(mhash); end - def message_saved?(multihash); store.message_saved?(multihash); end + def all_messages(mhash = nil) + store.all_messages(mhash) + end + + def message_saved?(multihash) + store.message_saved?(multihash) + end def _save_message(msg_obj) store.insert_message(Helpers.verify_message(self, msg_obj)) end - def read_message(multihash); store.read_message(multihash); end + def read_message(multihash) + store.read_message(multihash) + end def get_message_count_for(multihash) store.get_message_count_for(multihash) @@ -57,13 +81,14 @@ module Pigeon end # === DRAFTS - def reset_draft; add_config(CURRENT_DRAFT, nil); end + def reset_draft + add_config(CURRENT_DRAFT, nil) + end def new_draft(kind:, body: {}) old = get_config(CURRENT_DRAFT) - if old - raise "PUBLISH OR RESET CURRENT DRAFT (#{old.kind}) FIRST" - end + raise "PUBLISH OR RESET CURRENT DRAFT (#{old.kind}) FIRST" if old + save_draft(Draft.new(kind: kind, body: body)) end @@ -74,21 +99,22 @@ module Pigeon def get_draft draft = store.get_config(CURRENT_DRAFT) - if draft - return draft - else + unless draft raise "THERE IS NO DRAFT. CREATE ONE FIRST." end + draft end - def update_draft(k, v); Helpers.update_draft(self, k, v); end + def update_draft(k, v) + Helpers.update_draft(self, k, v) + end def reset_draft add_config(CURRENT_DRAFT, nil) end # Author a new message. - def publish_draft(draft = self.get_draft) + def publish_draft(draft = get_draft) Helpers.publish_draft(self, draft) end @@ -113,12 +139,16 @@ module Pigeon hash2filepath = Helpers.hash2file_path(mhash) blob_path = File.join(file_path, hash2filepath) blob = get_blob(mhash) - Helpers.write_to_disk(blob_path, mhash, blob) if blob + puts "=== EXPORT" + puts " blob_path: #{blob_path}" + puts " mhash: #{mhash}" + puts " blob: #{blob.length} bytes" + Helpers.write_to_disk(blob_path, mhash, blob) end # Render messages for all peers. content = messages - .map { |message| message.render } + .map(&:render) .join(BUNDLE_MESSAGE_SEPARATOR) File.write(File.join(file_path, "gossip.pgn"), content + CR) @@ -127,27 +157,38 @@ module Pigeon def import_bundle(file_path = DEFAULT_BUNDLE_PATH) bundle = File.read(File.join(file_path, "gossip.pgn")) tokens = Pigeon::Lexer.tokenize(bundle) - blobs = tokens.reduce(Set.new) do |set, (a, b, c)| + blobs = tokens.each_with_object(Set.new) do |(_a, b, c), set| [b, c].map do |d| set.add(d) if Helpers.blob_multihash?(d) end - set end.map do |multihash| - if !store.have_blob?(multihash) - binding.pry - end + binding.pry unless store.have_blob?(multihash) end Pigeon::Parser.parse(self, tokens) end # === BLOBS - def get_blob(b); store.get_blob(b); end - def add_blob(b); store.add_blob(b); end + def get_blob(b) + store.get_blob(b) + end + + def add_blob(b) + store.add_blob(b) + end # === DB Management - def get_config(k); store.get_config(k); end - def add_config(k, v); store.add_config(k, v); end - def reset_database; store.reset; init_ident; end + def get_config(k) + store.get_config(k) + end + + def add_config(k, v) + store.add_config(k, v) + end + + def reset_database + store.reset + init_ident + end private diff --git a/lib/pigeon/draft.rb b/lib/pigeon/draft.rb index a21862b..799163d 100644 --- a/lib/pigeon/draft.rb +++ b/lib/pigeon/draft.rb @@ -17,11 +17,12 @@ module Pigeon end def [](key) - self.body[key] + body[key] end def []=(key, value) raise STRING_KEYS_ONLY unless key.is_a?(String) + case value[0] when BLOB_SIGIL, MESSAGE_SIGIL, IDENTITY_SIGIL, STRING_SIGIL body[key] = value diff --git a/lib/pigeon/local_identity.rb b/lib/pigeon/local_identity.rb index 8f62183..29b3bb2 100644 --- a/lib/pigeon/local_identity.rb +++ b/lib/pigeon/local_identity.rb @@ -27,7 +27,7 @@ module Pigeon def sign(string) hex = @signing_key.sign(string) b64 = Helpers.b32_encode(hex) - return b64 + SIG_FOOTER + b64 + SIG_FOOTER end end end diff --git a/lib/pigeon/message.rb b/lib/pigeon/message.rb index 0fa2b7f..7da114e 100644 --- a/lib/pigeon/message.rb +++ b/lib/pigeon/message.rb @@ -9,7 +9,7 @@ module Pigeon end def multihash - tpl = self.render + tpl = render digest = Digest::SHA256.digest(tpl) sha256 = Helpers.b32_encode(digest) "#{MESSAGE_SIGIL}#{sha256}#{BLOB_FOOTER}" @@ -23,6 +23,7 @@ module Pigeon lipmaa:, signature:) raise MISSING_BODY if body.empty? + @author = author @body = body @depth = depth diff --git a/lib/pigeon/parser.rb b/lib/pigeon/parser.rb index 40ea00c..51ada7b 100644 --- a/lib/pigeon/parser.rb +++ b/lib/pigeon/parser.rb @@ -4,7 +4,8 @@ module Pigeon def self.parse(db, tokens) raise "NO!" unless db.is_a?(Pigeon::Database) - self.new(db, tokens).parse + + new(db, tokens).parse end def initialize(db, tokens) @@ -14,8 +15,8 @@ module Pigeon @results = [] end - def parse() - @tokens.each_with_index do |token, i| + def parse + @tokens.each_with_index do |token, _i| case token.first when :AUTHOR then set(:author, token.last) when :KIND then set(:kind, token.last) diff --git a/lib/pigeon/storage.rb b/lib/pigeon/storage.rb index 88dfec2..fd980ef 100644 --- a/lib/pigeon/storage.rb +++ b/lib/pigeon/storage.rb @@ -47,15 +47,17 @@ module Pigeon write do a = store.fetch(CONF_NS) raise "FIX SAVED DRAFTS" if value.instance_variable_get(:@db) + a[key] = value end end def add_blob(data) size = data.bytesize - if (size > BLOB_BYTE_LIMIT) + if size > BLOB_BYTE_LIMIT raise "Blob size limit is #{BLOB_BYTE_LIMIT} bytes. Got #{size}" end + raw_digest = Digest::SHA256.digest(data) b32_hash = Helpers.b32_encode(raw_digest) multihash = [BLOB_SIGIL, b32_hash, BLOB_FOOTER].join("") @@ -66,16 +68,15 @@ module Pigeon def get_blob(blob_multihash) path = File.join(Helpers.hash2file_path(blob_multihash)) path = File.join(PIGEON_BLOB_PATH, path) - if File.file?(path) - File.read(path) - else - nil - end + File.read(path) if File.file?(path) end # `nil` means "none" def get_message_count_for(mhash) - raise "Expected string, got #{mhash.class}" unless mhash.is_a?(String) # Delete later + unless mhash.is_a?(String) + raise "Expected string, got #{mhash.class}" + end # Delete later + read { store[COUNT_INDEX_NS][mhash] || 0 } end @@ -85,18 +86,21 @@ module Pigeon depth = -1 last = "" # TODO: This loop may become unresponsive. - until (last == nil) || (depth > 99999) - last = self.get_message_by_depth(author, depth += 1) + until last.nil? || (depth > 99_999) + last = get_message_by_depth(author, depth += 1) all.push(last) if last end - return all + all else read { store["messages"].keys } end end def get_message_by_depth(multihash, depth) - raise "Expected string, got #{multihash.class}" unless multihash.is_a?(String) # Delete later + unless multihash.is_a?(String) + raise "Expected string, got #{multihash.class}" + end # Delete later + # Map<[multihash(str), depth(int)], Signature> key = [multihash, depth].join(".") read { store[MESSAGE_BY_DEPTH_NS][key] } @@ -108,12 +112,10 @@ module Pigeon def insert_message(msg) write do - if store[MESG_NS].fetch(msg.multihash, false) - return msg - end + return msg if store[MESG_NS].fetch(msg.multihash, false) if store[BLCK_NS].member?(msg.author.multihash) - STDERR.puts("Blocked peer: #{msg.author.multihash}") + warn("Blocked peer: #{msg.author.multihash}") return msg end @@ -178,8 +180,16 @@ module Pigeon store.transaction(is_read_only) { yield } end - def write(&blk); transaction(false, &blk); end - def read(&blk); transaction(true, &blk); end - def on_disk?; File.file?(path); end + def write(&blk) + transaction(false, &blk) + end + + def read(&blk) + transaction(true, &blk) + end + + def on_disk? + File.file?(path) + end end end diff --git a/pigeon.gemspec b/pigeon.gemspec index 7e2091b..70c39c7 100644 --- a/pigeon.gemspec +++ b/pigeon.gemspec @@ -12,6 +12,6 @@ Gem::Specification.new do |s| s.homepage = "https://tildegit.org/PigeonProtocolConsortium/pigeon_ruby" s.license = "GPL-3.0-or-later" s.executables = "pigeon-cli" - s.add_runtime_dependency "thor", "~> 0.20", ">= 0.20.3" s.add_runtime_dependency "ed25519", "~> 1.2", ">= 1.2.4" + s.add_runtime_dependency "thor", "~> 0.20", ">= 0.20.3" end diff --git a/spec/fixtures/has_blobs/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG b/spec/fixtures/has_blobs/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG new file mode 100644 index 0000000..38b1f66 Binary files /dev/null and b/spec/fixtures/has_blobs/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG/622PRNJ/7C0S05X/R2AHDPK/WMG051B/1QW5SXM/N2RQHF2/AND6J8V.GPG differ diff --git a/spec/fixtures/has_blobs/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG b/spec/fixtures/has_blobs/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG new file mode 100644 index 0000000..ed902d4 Binary files /dev/null and b/spec/fixtures/has_blobs/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG/FV0FJ0Y/ZADY7C5/JTTFYPK/DBHTZJ5/JVVP5TC/KP0605W/WXYJG4V.MRG differ diff --git a/spec/fixtures/has_blobs/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG b/spec/fixtures/has_blobs/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG new file mode 100644 index 0000000..a4cd0bd Binary files /dev/null and b/spec/fixtures/has_blobs/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG/YPF11E5/N9JFVB6/KB1N1WD/VVT9DXM/CHE0XJW/BZHT2CQ/29S5SEP.CSG differ diff --git a/spec/pigeon/bundle_spec.rb b/spec/pigeon/bundle_spec.rb index 637b726..c180847 100644 --- a/spec/pigeon/bundle_spec.rb +++ b/spec/pigeon/bundle_spec.rb @@ -18,8 +18,8 @@ RSpec.describe Pigeon::Message do db.add_message("a", { "b" => db.add_blob("three") })] normal = (1..10) .to_a - .map do |n| { "foo" => ["bar", "123", SecureRandom.uuid].sample } end - .map do |d| db.add_message(SecureRandom.uuid, d) end + .map { |_n| { "foo" => ["bar", "123", SecureRandom.uuid].sample } } + .map { |d| db.add_message(SecureRandom.uuid, d) } blobs + normal end @@ -51,7 +51,7 @@ RSpec.describe Pigeon::Message do db.add_blob(File.read("b.gif")) => db.add_blob(File.read("c.gif")), }) db.export_bundle("./spec/fixtures/has_blobs") - STDERR.puts("The directory structure is not correct.") + warn("The directory structure is not correct.") exit(1) db.import_bundle("./spec/fixtures/has_blobs") expect(db.all_messages.count).to eq(0) diff --git a/spec/pigeon/lexer_spec.rb b/spec/pigeon/lexer_spec.rb index 944d013..69963a3 100644 --- a/spec/pigeon/lexer_spec.rb +++ b/spec/pigeon/lexer_spec.rb @@ -102,7 +102,7 @@ RSpec.describe Pigeon::Lexer do [:BODY_END], [:SIGNATURE, "BBE732XXZ33XTCW1CRA9RG13FQ0FVMR61SAHD621VH8C64B4WA8C86JSTTAHG4CSGNBJJ7YSAVRF3YEBX6GTEB6RRWGDA84VJZPMR3R.sig.ed25519"], [:MESSAGE_END], - ] + ].freeze MESSAGE_LINES = [ "author @VG44QCHKA38E7754RQ5DAFBMMD2CCZQRZ8BR2J4MRHHGVTHGW670.ed25519", @@ -131,7 +131,7 @@ RSpec.describe Pigeon::Lexer do it "tokenizes a bundle" do bundle = File.read("./spec/fixtures/normal/gossip.pgn") tokens = Pigeon::Lexer.tokenize(bundle) - EXPECTED_TOKENS1.each_with_index do |item, i| + EXPECTED_TOKENS1.each_with_index do |_item, i| expect(tokens[i]).to eq(EXPECTED_TOKENS1[i]) end end @@ -139,7 +139,7 @@ RSpec.describe Pigeon::Lexer do it "tokenizes a single message" do string = message.render tokens = Pigeon::Lexer.tokenize(string) - hash = tokens.reduce({ BODY: {} }) do |h, token| + hash = tokens.each_with_object({ BODY: {} }) do |token, h| case token.first when :HEADER_END, :BODY_END, :MESSAGE_END h @@ -148,7 +148,6 @@ RSpec.describe Pigeon::Lexer do else h[token.first] = token.last end - h end expect(hash[:AUTHOR]).to eq(message.author.multihash) diff --git a/spec/pigeon/message_spec.rb b/spec/pigeon/message_spec.rb index 9e57f6b..aae46bc 100644 --- a/spec/pigeon/message_spec.rb +++ b/spec/pigeon/message_spec.rb @@ -157,7 +157,7 @@ RSpec.describe Pigeon::Message do kind[rand(0...8)] = n db.reset_draft db.new_draft(kind: kind) - boom = ->() { db.publish_draft.render } + boom = -> { db.publish_draft.render } expect(boom).to raise_error(Pigeon::Lexer::LexError) end end @@ -169,7 +169,7 @@ RSpec.describe Pigeon::Message do key = SecureRandom.alphanumeric(8) key[rand(0...8)] = n db.update_draft(key, "should crash") - boom = ->() { Pigeon::Lexer.tokenize(db.publish_draft.render) } + boom = -> { Pigeon::Lexer.tokenize(db.publish_draft.render) } expect(boom).to raise_error(Pigeon::Lexer::LexError) end end diff --git a/spec/pigeon/parser_spec.rb b/spec/pigeon/parser_spec.rb index 5cae952..d767624 100644 --- a/spec/pigeon/parser_spec.rb +++ b/spec/pigeon/parser_spec.rb @@ -20,7 +20,7 @@ RSpec.describe Pigeon::Lexer do [:BODY_ENTRY, "duplicate", "This key is a duplicate."], [:SIGNATURE, "DN7yPTE-m433ND3jBL4oM23XGxBKafjq0Dp9ArBQa_TIGU7DmCxTumieuPBN-NKxlx_0N7-c5zjLb5XXVHYPCQ==.sig.ed25519"], [:MESSAGE_END], - ] + ].freeze it "parses tokens" do results = Pigeon::Parser.parse(db, tokens) diff --git a/spec/pigeon/storage_spec.rb b/spec/pigeon/storage_spec.rb index 27444ac..1f22dc7 100644 --- a/spec/pigeon/storage_spec.rb +++ b/spec/pigeon/storage_spec.rb @@ -2,8 +2,8 @@ require "spec_helper" RSpec.describe Pigeon::Storage do LOGO_BLOB = File.read("./logo.png") - IDS = %w(@ZMWM5PSXRN7RFRMSWW1E3V5DNGC4XGGJTHKCAGB48SNRG4XXE5NG.ed25519 - @VF0Q4KXQNY6WCAXF17GAZGDPAX8XKM70SB8N7V0NSD1H370ZCJBG.ed25519) + IDS = %w[@ZMWM5PSXRN7RFRMSWW1E3V5DNGC4XGGJTHKCAGB48SNRG4XXE5NG.ed25519 + @VF0Q4KXQNY6WCAXF17GAZGDPAX8XKM70SB8N7V0NSD1H370ZCJBG.ed25519].freeze let(:db) do db = Pigeon::Database.new diff --git a/spec/pigeon/template_spec.rb b/spec/pigeon/template_spec.rb index a854fcf..c661539 100644 --- a/spec/pigeon/template_spec.rb +++ b/spec/pigeon/template_spec.rb @@ -1,7 +1,7 @@ require "spec_helper" RSpec.describe Pigeon::MessageSerializer do - SHIM_ATTRS = [:author, :body, :kind, :depth, :prev, :signature, :lipmaa] + SHIM_ATTRS = %i[author body kind depth prev signature lipmaa].freeze MessageShim = Struct.new(*SHIM_ATTRS) TOP_HALF = ["author FAKE_AUTHOR", "\nkind FAKE_KIND", diff --git a/tutorial.rb b/tutorial.rb index 55d6369..fa482ce 100644 --- a/tutorial.rb +++ b/tutorial.rb @@ -27,7 +27,7 @@ require_relative "lib/pigeon" require "pry" -files = %w(a.gif b.gif c.gif) +files = %w[a.gif b.gif c.gif] body = { "what" => "A simple bundle with a few blobs" } db = Pigeon::Database.new(path: "new.db") db.add_message("description", body)