From a3c4d86025b06a3c4f743d9a8996537af3c5e936 Mon Sep 17 00:00:00 2001 From: Rick Carlino Date: Wed, 22 Apr 2020 08:21:55 -0500 Subject: [PATCH] Add blobs + peer messages to bundle --- .gitignore | 1 + README.md | 15 +++++++++------ lib/pigeon.rb | 33 +++++++++++++++++++++++++-------- lib/pigeon/database.rb | 15 +++++++-------- lib/pigeon/storage.rb | 19 ++++--------------- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index e4e35ce..16243e6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ doc/ *scratchpad* pigeon.bundle *.gem +bundle/ \ No newline at end of file diff --git a/README.md b/README.md index 8b92761..7ab2077 100644 --- a/README.md +++ b/README.md @@ -89,16 +89,19 @@ TODO - [X] Add log count to `pigeon-cli status` - [X] Delete `Draft#put` entirely. - [X] Check block list before ingesting bundles. - - [ ] Need a way of importing / exporting a feeds blobs. (see "Bundle Brainstorming" below) + - [X] Need a way of importing / exporting a feeds blobs. (see "Bundle Brainstorming" below) + - [X] Need a way of adding peers messages / gossip to bundles. (see "Bundle Brainstorming" below) - [ ] Update README.md / tutorial.rb (user manual for `Pigeon::Database`). - - [ ] Make the switch to LevelDB, RocksDB, [UNQLite](https://unqlite.org/features.html) or similar (currently using Ruby PStore). - - [ ] Need a way of adding peers messages / gossip to bundles. (see "Bundle Brainstorming" below) + - [ ] Update spec document CLI usage examples to reflect API changes in 2020. + - [ ] Publish to RubyGems + +# Optimizations - [ ] add parsers and validators for all CLI inputs + - [ ] Make the switch to LevelDB, RocksDB, [UNQLite](https://unqlite.org/features.html) or similar (currently using Ruby PStore). + - [ ] Convert literals to constants, remove unused locals, reduce duplication. - [ ] Reduce whole darn repo into single module to aide portability. `::Helpers` module is OK. - [ ] Update the bundles.md document once `bundle consume` works. - [ ] 100% documentation - - [ ] Update spec document CLI usage examples to reflect API changes in 2020. - - [ ] Publish to RubyGems - [ ] Performance benchmarks (Do this second to last!) - [ ] Performance tuning (Do this last!) @@ -128,7 +131,7 @@ Here's how we will support that: * All messages are expected to be sorted by depth * Messages from multiple authors may be included in a single bundle, but the messages must appear in the correct order with regards to the `depth` field. 3. Blobs are stored in a very specific hierarchy to maintain FAT compatibility: - * `blobs/sha256/AAAAAAAA/BBBBBBBB/CCCCCCCC/DDDDDDDD/EEEEEEEE/FFFFFFFF/G.HHH` + * `blobs/bundle/7Z2CSZK/MB1RE5G/6SKXRZ6/3ZGCNP8/VVEM3K0/XFMYKET/RDQSM5W.BSG` Additional notes: diff --git a/lib/pigeon.rb b/lib/pigeon.rb index e5c19c9..15d7df7 100644 --- a/lib/pigeon.rb +++ b/lib/pigeon.rb @@ -210,15 +210,32 @@ module Pigeon Dir.mkdir(path) unless Dir.exists?(path) end - def self.split_file_path(b32_hash) + def self.write_to_disk(base_path, mhash, data) + p = Helpers.hash2file_path(mhash) + file_name = p.pop + Helpers.mkdir_p(base_path) + dir = p.reduce(base_path) do |accum, item| + path = File.join(accum, item) + Helpers.mkdir_p(path) + path + end + full_path = File.join(dir, file_name) + unless File.file?(full_path) + File.write(full_path, data) + end + end + + def self.hash2file_path(mhash) + mhash = mhash.sub("&", "") + [ - b32_hash[0], - b32_hash[1...9], - b32_hash[9...17], - b32_hash[17...25], - b32_hash[25...33], - b32_hash[33...41], - [b32_hash[41...49], ".", b32_hash[49...52]].join(""), + mhash[0...7], + mhash[7...14], + mhash[14...21], + mhash[21...28], + mhash[28...35], + mhash[35...42], + [mhash[42...49], ".", mhash[49...52]].join(""), ] end diff --git a/lib/pigeon/database.rb b/lib/pigeon/database.rb index 3eb95f6..e8c4acb 100644 --- a/lib/pigeon/database.rb +++ b/lib/pigeon/database.rb @@ -106,15 +106,14 @@ module Pigeon content = messages .map { |message| message.render } .join(BUNDLE_MESSAGE_SEPARATOR) - - # MKdir - Helpers.mkdir_p("bundle") - # Get blobs for _all_ peers - blobs = messages.map(&:collect_blobs).flatten.uniq - # binding.pry if blobs.any? - # Write bundle to dir - # Link blobs to dir File.write(file_path, content + CR) + + messages + .map(&:collect_blobs) + .flatten + .uniq + .map { |mhash| ["bundle", mhash, get_blob(mhash)] } + .map { |arg| Helpers.write_to_disk(*arg) } end def ingest_bundle(file_path = DEFAULT_BUNDLE_PATH) diff --git a/lib/pigeon/storage.rb b/lib/pigeon/storage.rb index ba47ff5..711abc9 100644 --- a/lib/pigeon/storage.rb +++ b/lib/pigeon/storage.rb @@ -59,12 +59,12 @@ module Pigeon raw_digest = Digest::SHA256.digest(data) b32_hash = Helpers.b32_encode(raw_digest) multihash = [BLOB_SIGIL, b32_hash, BLOB_FOOTER].join("") - write_to_disk(b32_hash, data) + write_to_disk(multihash, data) multihash end def get_blob(blob_multihash) - path = File.join(Helpers.split_file_path(blob_multihash[1..52])) + path = File.join(Helpers.hash2file_path(blob_multihash)) path = File.join(PIGEON_BLOB_PATH, path) if File.file?(path) File.read(path) @@ -132,23 +132,12 @@ module Pigeon private - def write_to_disk(b32_hash, data) - p = Helpers.split_file_path(b32_hash) - file_name = p.pop - dir = p.reduce(PIGEON_BLOB_PATH) do |accum, item| - path = File.join(accum, item) - Helpers.mkdir_p(path) - path - end - full_path = File.join(dir, file_name) - unless File.file?(full_path) - File.write(full_path, data) - end + def write_to_disk(mhash, data) + Helpers.write_to_disk(PIGEON_BLOB_PATH, mhash, data) end def bootstrap write do - # TODO: Why is there a depth and count index?? store[BLCK_NS] ||= Set.new store[CONF_NS] ||= {} store[COUNT_INDEX_NS] ||= {}