From 780823ccafb60868d6e8f69cbfa251080030ab93 Mon Sep 17 00:00:00 2001 From: Rick Carlino Date: Thu, 16 Apr 2020 09:05:10 -0500 Subject: [PATCH] [REALLY UNSTABLE] Started removal of Pigeon::Storage.current --- README.md | 10 ++++- dist/pigeon.rb | 6 --- dist/pigeon/database.rb | 50 +++++++++++++++++++++++-- dist/pigeon/draft.rb | 22 ++--------- dist/pigeon/local_identity.rb | 20 +--------- dist/pigeon/message.rb | 28 ++++++++------ dist/pigeon/storage.rb | 17 ++++++--- pigeon-cli | 4 +- spec/pigeon/bundle_spec.rb | 6 +-- spec/pigeon/draft_spec.rb | 19 +++++----- spec/pigeon/lexer_spec.rb | 13 +++---- spec/pigeon/local_identity_spec.rb | 5 --- spec/pigeon/message_spec.rb | 25 ++++++------- spec/pigeon/parser_spec.rb | 5 --- spec/pigeon/storage_spec.rb | 60 +++++++++++++++--------------- 15 files changed, 149 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index fcae400..9c8f9ee 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,16 @@ Eg: `pigeon identity show` becomes `./pigeon-cli show`. - [X] Make all methods private except those required for the CLI. - [X] Add Lipmaa links like the Bamboo folks do. - [X] Set a max message size. + - [ ] Clean up all singletons / .current hack from storage.rb + - [ ] Clean up all singletons / .current hack from parser.rb + - [ ] Clean up all singletons / .current hack from lexer.rb + - [ ] Clean up all singletons / .current hack from message.rb + - [ ] Clean up all singletons / .current hack from draft_serializer.rb + - [ ] Clean up all singletons / .current hack from message_serializer.rb + - [ ] Clean up all singletons / .current hack from remote_identity.rb + - [ ] Clean up all singletons / .current hack from draft.rb + - [ ] Clean up all singletons / .current hack from local_identity.rb - [ ] Update README.md. Needs user manual for new `Pigeon::Database` class. - - [ ] Remove all `.current` "singletons" / hacks - [ ] Make the switch to LevelDB, RocksDB, [UNQLite](https://unqlite.org/features.html) or similar (currently using Ruby PStore). - [ ] Make CLI names consistent with API names. Eg: find vs. read. - [ ] Create regexes in ::Lexer using strings and Regexp.new() for cleaner regexes. diff --git a/dist/pigeon.rb b/dist/pigeon.rb index 8422fbf..40e363d 100644 --- a/dist/pigeon.rb +++ b/dist/pigeon.rb @@ -135,12 +135,6 @@ module Pigeon .join("") end - def self.create_message(kind, params) - draft = Pigeon::Draft.create(kind: kind) - params.map { |(k, v)| draft[k] = v } - draft.publish - end - def self.verify_string(identity, string_signature, string) binary_signature = decode_multihash(string_signature) diff --git a/dist/pigeon/database.rb b/dist/pigeon/database.rb index 6b5b0cd..feb11f2 100644 --- a/dist/pigeon/database.rb +++ b/dist/pigeon/database.rb @@ -1,11 +1,28 @@ module Pigeon class Database - def initialize + attr_reader :local_identity + + def initialize(path: PIGEON_DB_PATH, + seed: SecureRandom.random_bytes(Ed25519::KEY_SIZE)) + @store = Pigeon::Storage.new(path: path) + init_local_identity(seed) + end + + def find_all; store.find_all; end + def put_blob(b); store.put_blob(b); end + def set_config(k, v); store.set_config(k, v); end + def get_config(k); store.get_config(k); end + def reset_current_draft; set_config(CURRENT_DRAFT, nil); end + def reset; store.reset; end + + def create_message(kind, params) + draft = Pigeon::Draft.new(kind: kind, db: self) + params.map { |(k, v)| draft[k] = v } + draft.publish end def create_bundle(file_path = DEFAULT_BUNDLE_PATH) - s = Pigeon::Storage.current - content = s + content = store .find_all(Pigeon::LocalIdentity.current.multihash) .map { |multihash| s.read_message(multihash) } .sort_by(&:depth) @@ -19,5 +36,32 @@ module Pigeon tokens = Pigeon::Lexer.tokenize(bundle) Pigeon::Parser.parse(tokens) end + + def create_draft(kind:, body: {}) + save_draft(Draft.new(kind: kind, body: body)) + end + + def save_draft(draft) + set_config(CURRENT_DRAFT, draft) + draft + end + + def current_draft + store.get_config(CURRENT_DRAFT) or raise NO_DRAFT_FOUND + end + + private + + attr_reader :store + + def init_local_identity(new_seed) + key = store.get_config(SEED_CONFIG_KEY) + if key + @local_identity = LocalIdentity.new(key) + else + @local_identity = LocalIdentity.new(new_seed) + set_config(SEED_CONFIG_KEY, new_seed) + end + end end end diff --git a/dist/pigeon/draft.rb b/dist/pigeon/draft.rb index 393b23b..bf8160e 100644 --- a/dist/pigeon/draft.rb +++ b/dist/pigeon/draft.rb @@ -5,25 +5,14 @@ module Pigeon attr_reader :signature, :prev, :lipmaa, :kind, :internal_id, :depth, :body, :author - def self.create(kind:, body: {}) - self.new(kind: kind, body: body).save_as_draft - end - - def self.current - Pigeon::Storage.current.get_config(CURRENT_DRAFT) or raise NO_DRAFT_FOUND - end - - def self.reset_current - Pigeon::Storage.current.set_config(CURRENT_DRAFT, nil) - end - def discard if Draft.current&.internal_id == @internal_id Draft.reset_current end end - def initialize(kind:, body: {}) + def initialize(kind:, body: {}, db:) + @db = db @signature = Pigeon::NOTHING @prev = Pigeon::NOTHING @kind = kind @@ -50,15 +39,10 @@ module Pigeon # This might be a bad or good idea. Not sure yet. self.body[key] = value.inspect end - self.save_as_draft + @db.save_draft(self) return self.body[key] end - def save_as_draft - Pigeon::Storage.current.set_config(CURRENT_DRAFT, self) - self - end - # Author a new message. def publish template = MessageSerializer.new(self) diff --git a/dist/pigeon/local_identity.rb b/dist/pigeon/local_identity.rb index 2dab4d1..8f62183 100644 --- a/dist/pigeon/local_identity.rb +++ b/dist/pigeon/local_identity.rb @@ -6,22 +6,9 @@ module Pigeon # help us maintain our sanity when the Gem's API # changes. class LocalIdentity - def self.reset - @current = nil - end - - def self.current - if @current - @current - else - key = Pigeon::Storage.current.get_config(SEED_CONFIG_KEY) - @current = (key ? self.new(key) : self.new).save! - end - end - # `seed` is a 32-byte seed value from which # the key should be derived - def initialize(seed = SecureRandom.random_bytes(Ed25519::KEY_SIZE)) + def initialize(seed) @seed = seed @signing_key = Ed25519::SigningKey.new(@seed) end @@ -42,10 +29,5 @@ module Pigeon b64 = Helpers.b32_encode(hex) return b64 + SIG_FOOTER end - - def save! - Pigeon::Storage.current.set_config(SEED_CONFIG_KEY, @seed) - self - end end end diff --git a/dist/pigeon/message.rb b/dist/pigeon/message.rb index 765fb83..50abb50 100644 --- a/dist/pigeon/message.rb +++ b/dist/pigeon/message.rb @@ -11,14 +11,22 @@ module Pigeon MSG_SIZE_ERROR = "Messages cannot have more than 64 keys. Got %s." # Store a message that someone (not the LocalIdentity) # has authored. - def self.ingest(author:, body:, depth:, kind:, lipmaa:, prev:, signature:) + def self.ingest(author:, + body:, + depth:, + kind:, + lipmaa:, + prev:, + signature:, + db:) params = { author: RemoteIdentity.new(author), kind: kind, body: body, prev: prev, lipmaa: lipmaa, signature: signature, - depth: depth } + depth: depth, + db: db } # Kind of weird to use `send` but #save! is private, # and I don't want people calling it directly without going through the # lexer / parser first. @@ -39,11 +47,11 @@ module Pigeon private def save! - return store.read_message(multihash) if store.message?(multihash) + return db.read_message(multihash) if db.message?(multihash) verify_counted_fields verify_signature self.freeze - store.save_message(self) + db.save_message(self) self end @@ -60,8 +68,8 @@ module Pigeon msg = MSG_SIZE_ERROR % key_count raise MessageSizeError, msg end - count = store.get_message_count_for(author.multihash) - expected_prev = store.get_message_by_depth(author.multihash, count - 1) || Pigeon::NOTHING + count = db.get_message_count_for(author.multihash) + expected_prev = db.get_message_by_depth(author.multihash, count - 1) || Pigeon::NOTHING assert("depth", count, depth) # TODO: Re-visit this. Our current verification method # is probably too strict and won't allow for partial @@ -81,8 +89,10 @@ module Pigeon depth:, prev:, lipmaa:, - signature:) + signature:, + db:) raise MISSING_BODY if body.empty? + @db = db @author = author @body = body @depth = depth @@ -95,9 +105,5 @@ module Pigeon def template MessageSerializer.new(self) end - - def store - Pigeon::Storage.current - end end end diff --git a/dist/pigeon/storage.rb b/dist/pigeon/storage.rb index a7840e7..7ce1230 100644 --- a/dist/pigeon/storage.rb +++ b/dist/pigeon/storage.rb @@ -2,12 +2,15 @@ require "pstore" module Pigeon class Storage - def self.current - @current ||= self.new + attr_reader :path + + def initialize(path: PIGEON_DB_PATH) + @path = path + bootstrap unless bootstrapped? end - def self.reset - File.delete(PIGEON_DB_PATH) if File.file?(PIGEON_DB_PATH) + def reset + File.delete(path) if bootstrapped? @current = nil end @@ -45,7 +48,7 @@ module Pigeon write { store[CONF_NS][key] = value } end - def set_blob(data) + def put_blob(data) raw_digest = Digest::SHA256.digest(data) b64_digest = Helpers.b32_encode(raw_digest) multihash = [BLOB_SIGIL, b64_digest, BLOB_FOOTER].join("") @@ -146,5 +149,9 @@ module Pigeon def write(&blk); transaction(false, &blk); end def read(&blk); transaction(true, &blk); end + + def bootstrapped? + File.file?(path) + end end end diff --git a/pigeon-cli b/pigeon-cli index c37df6b..fef9e58 100755 --- a/pigeon-cli +++ b/pigeon-cli @@ -47,7 +47,7 @@ module Pigeon def set(data = "") blob = (data != "") ? data : STDIN.read - puts Pigeon::Storage.current.set_blob(blob) + puts Pigeon::Storage.current.put_blob(blob) end desc "get", "Read arbitrary data from the database" @@ -87,7 +87,7 @@ module Pigeon desc "create", "Begin a new Pigeon message" def create(kind) - puts Pigeon::Draft.create(kind: kind).render_as_draft + puts db.create_draft(kind: kind).render_as_draft end desc "append", "Add a key/value pair to the current DRAFT" diff --git a/spec/pigeon/bundle_spec.rb b/spec/pigeon/bundle_spec.rb index 92988fb..cb919d3 100644 --- a/spec/pigeon/bundle_spec.rb +++ b/spec/pigeon/bundle_spec.rb @@ -4,17 +4,15 @@ RSpec.describe Pigeon::Message do before(:each) do p = Pigeon::DEFAULT_BUNDLE_PATH File.delete(p) if File.file?(p) - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset end - let(:db) do Pigeon::Database.new end + let(:db) { Pigeon::Database.new } def create_fake_messages (1..10) .to_a .map do |n| { "foo" => ["bar", "123", SecureRandom.uuid].sample } end - .map do |d| Pigeon::Helpers.create_message(SecureRandom.uuid, d) end + .map do |d| db.create_message(SecureRandom.uuid, d) end end it "creates a bundle" do diff --git a/spec/pigeon/draft_spec.rb b/spec/pigeon/draft_spec.rb index 28d0fae..77950b9 100644 --- a/spec/pigeon/draft_spec.rb +++ b/spec/pigeon/draft_spec.rb @@ -1,18 +1,19 @@ require "spec_helper" RSpec.describe Pigeon::Draft do + let(:db) do + Pigeon::Database.new + end + let(:message) do - message = Pigeon::Draft.create(kind: "unit_test") - hash = Pigeon::Storage.current.set_blob(File.read("./logo.png")) + message = db.create_draft(kind: "unit_test") + hash = db.put_blob(File.read("./logo.png")) message["a"] = "bar" message["b"] = hash message end - before(:each) do - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset - end + before(:each) { db.reset } MSG = [ "author DRAFT", @@ -26,15 +27,15 @@ RSpec.describe Pigeon::Draft do ].join("\n") it "renders a message" do - pk = Pigeon::LocalIdentity.current.multihash + pk = db.local_identity.multihash actual = message.render_as_draft expected = MSG.gsub("___", pk) expect(actual).to start_with(expected) end it "creates a new message" do - message = Pigeon::Draft.create(kind: "unit_test") - hash = Pigeon::Storage.current.set_blob(File.read("./logo.png")) + message = db.create_draft(kind: "unit_test") + hash = db.put_blob(File.read("./logo.png")) expectations = { kind: "unit_test", body: { diff --git a/spec/pigeon/lexer_spec.rb b/spec/pigeon/lexer_spec.rb index ae55d4d..83a026e 100644 --- a/spec/pigeon/lexer_spec.rb +++ b/spec/pigeon/lexer_spec.rb @@ -115,15 +115,14 @@ RSpec.describe Pigeon::Lexer do "signature hHvhdvUcrabhFPz52GSGa9_iuudOsGEEE7S0o0WJLqjQyhLfgUy72yppHXsG6T4E21p6EEI6B3yRcjfurxegCA==.sig.ed25519", ].freeze - let(:message) do - draft = Pigeon::Draft.create(kind: "unit_test") - draft["foo"] = "bar" - draft.publish + let(:db) do + Pigeon::Database.new end - before(:each) do - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset + let(:message) do + draft = db.create_draft(kind: "unit_test") + draft["foo"] = "bar" + draft.publish end it "tokenizes a bundle" do diff --git a/spec/pigeon/local_identity_spec.rb b/spec/pigeon/local_identity_spec.rb index 78df260..b70336c 100644 --- a/spec/pigeon/local_identity_spec.rb +++ b/spec/pigeon/local_identity_spec.rb @@ -32,9 +32,4 @@ RSpec.describe Pigeon::LocalIdentity do result = Pigeon::Helpers.decode_multihash(example) expect(result).to eq(Pigeon::Helpers.b32_decode(whatever)) end - - it "caches LocalIdentity.current" do - first_kp = Pigeon::LocalIdentity.current - expect(Pigeon::LocalIdentity.current).to be(first_kp) # Need strict equality here! - end end diff --git a/spec/pigeon/message_spec.rb b/spec/pigeon/message_spec.rb index 4c700ff..9d2bd76 100644 --- a/spec/pigeon/message_spec.rb +++ b/spec/pigeon/message_spec.rb @@ -1,13 +1,8 @@ require "spec_helper" RSpec.describe Pigeon::Message do - before(:each) do - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset - end - def create_draft(params) - draft = Pigeon::Draft.create(kind: "unit_test") + draft = db.create_draft(kind: "unit_test") params.each { |(k, v)| draft[k] = v } draft end @@ -17,10 +12,13 @@ RSpec.describe Pigeon::Message do draft.publish end + let(:db) do + Pigeon::Database.new + end + let(:draft) do - hash = Pigeon::Storage.current.set_blob(File.read("./logo.png")) - create_draft({ "a" => "bar", - "b" => hash }) + hash = db.put_blob(File.read("./logo.png")) + create_draft({ "a" => "bar", "b" => hash }) end let(:templated_message) { create_message({ "a" => "b" }) } @@ -65,7 +63,7 @@ RSpec.describe Pigeon::Message do it "creates a chain of messages" do all = [] 0.upto(4) do |expected_depth| - draft1 = Pigeon::Draft.create(kind: "unit_test") + draft1 = db.create_draft(kind: "unit_test") draft1["description"] = "Message number #{expected_depth}" message = draft1.publish all.push(message) @@ -106,8 +104,7 @@ RSpec.describe Pigeon::Message do it "verifies accuracy of signatures" do # === Initial setup - Pigeon::LocalIdentity.current - secret = Pigeon::Storage.current.get_config(Pigeon::SEED_CONFIG_KEY) + secret = db.get_config(Pigeon::SEED_CONFIG_KEY) message = templated_message plaintext = template.render_without_signature @@ -153,7 +150,7 @@ RSpec.describe Pigeon::Message do WHITESPACE.map do |n| kind = SecureRandom.alphanumeric(8) kind[rand(0...8)] = n - draft = Pigeon::Draft.create(kind: kind) + draft = db.create_draft(kind: kind) draft["body"] = "empty" boom = ->() { Pigeon::Lexer.tokenize(draft.publish.render) } expect(boom).to raise_error(Pigeon::Lexer::LexError) @@ -162,7 +159,7 @@ RSpec.describe Pigeon::Message do it "does not allow whitespace in key names" do WHITESPACE.map do |n| - draft = Pigeon::Draft.create(kind: "unit_test") + draft = db.create_draft(kind: "unit_test") key = SecureRandom.alphanumeric(8) key[rand(0...8)] = n draft[key] = "should crash" diff --git a/spec/pigeon/parser_spec.rb b/spec/pigeon/parser_spec.rb index defda68..9ebe8ed 100644 --- a/spec/pigeon/parser_spec.rb +++ b/spec/pigeon/parser_spec.rb @@ -1,11 +1,6 @@ require "spec_helper" RSpec.describe Pigeon::Lexer do - before(:each) do - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset - end - let(:db) { Pigeon::Database.new } let(:example_bundle) { File.read("./spec/fixtures/normal.bundle") } let(:tokens) { Pigeon::Lexer.tokenize(example_bundle) } diff --git a/spec/pigeon/storage_spec.rb b/spec/pigeon/storage_spec.rb index a8883aa..e9a2969 100644 --- a/spec/pigeon/storage_spec.rb +++ b/spec/pigeon/storage_spec.rb @@ -6,72 +6,70 @@ RSpec.describe Pigeon::Storage do @VF0Q4KXQNY6WCAXF17GAZGDPAX8XKM70SB8N7V0NSD1H370ZCJBG.ed25519) before(:each) do - Pigeon::Storage.reset - Pigeon::LocalIdentity.reset + db.reset end - let(:s) { Pigeon::Storage.current } let(:db) { Pigeon::Database.new } it "sets a config" do - s.set_config("FOO", "BAR") - value = s.get_config("FOO") + db.set_config("FOO", "BAR") + value = db.get_config("FOO") expect(value).to eq("BAR") - s.set_config("FOO", nil) - value = s.get_config("FOO") + db.set_config("FOO", nil) + value = db.get_config("FOO") expect(value).to eq(nil) end it "manages configs" do - s.set_config("FOO", "BAR") - value = s.get_config("FOO") + db.set_config("FOO", "BAR") + value = db.get_config("FOO") expect(value).to eq("BAR") end it "manages blobs" do - logo_hash = s.set_blob(LOGO_BLOB) - expect(s.get_blob(logo_hash)).to eq(LOGO_BLOB) + logo_hash = db.put_blob(LOGO_BLOB) + expect(db.get_blob(logo_hash)).to eq(LOGO_BLOB) end it "manages peers" do - s.add_peer(IDS[0]) - s.add_peer(IDS[1]) - expect(s.all_peers).to include(IDS[0]) - expect(s.all_peers).to include(IDS[1]) + db.add_peer(IDS[0]) + db.add_peer(IDS[1]) + expect(db.all_peers).to include(IDS[0]) + expect(db.all_peers).to include(IDS[1]) - s.remove_peer(IDS[0]) - expect(s.all_peers).not_to include(IDS[0]) - expect(s.all_blocks).not_to include(IDS[0]) + db.remove_peer(IDS[0]) + expect(db.all_peers).not_to include(IDS[0]) + expect(db.all_blocks).not_to include(IDS[0]) - s.block_peer(IDS[1]) - expect(s.all_peers).not_to include(IDS[1]) - expect(s.all_blocks).to include(IDS[1]) - expect(s.all_blocks.count).to eq(1) + db.block_peer(IDS[1]) + expect(db.all_peers).not_to include(IDS[1]) + expect(db.all_blocks).to include(IDS[1]) + expect(db.all_blockdb.count).to eq(1) end it "finds all authored by a particular feed" do ingested_messages = db.ingest_bundle("./spec/fixtures/normal.bundle") - author = ingested_messages.first.author.multihash - actual_messages = Pigeon::Storage.current.find_all(author) - search_results = Pigeon::Storage.current.find_all(author) + author = ingested_messagedb.first.author.multihash + actual_messages = db.find_all(author) + search_results = db.find_all(author) end it "finds all messages" do msgs = [ - Pigeon::Helpers.create_message("strings", { + db.create_message("strings", { "example_1.1" => "This is a string.", "example=_." => "A second string.", }), - Pigeon::Helpers.create_message("d", { - "e" => Pigeon::Storage.current.set_blob(File.read("./logo.png")), + db.create_message("d", { + "e" => db.put_blob(File.read("./logo.png")), }), - Pigeon::Helpers.create_message("g", { + db.create_message("g", { "me_myself_and_i" => Pigeon::LocalIdentity.current.multihash, }), ] me = Pigeon::LocalIdentity.current.multihash - results = Pigeon::Storage.current.find_all(me) - expect(results.length).to eq(3) + results = db.find_all(me) + expect(resultdb.length).to eq(3) expect(msgs[0].multihash).to eq(results[0]) expect(msgs[1].multihash).to eq(results[1]) expect(msgs[2].multihash).to eq(results[2])