Storage layer obeys blocklist
This commit is contained in:
parent
c4c4ddb3bd
commit
10dc8e7cab
14
README.md
14
README.md
|
@ -6,9 +6,10 @@ This is a WIP [Pigeon Protocol] client written in Ruby.
|
|||
|
||||
# Caveats
|
||||
|
||||
* Blobs are not included in bundles yet.
|
||||
* Not published to RubyGems yet (see installation instructions below)
|
||||
* Not thread safe. This particular implementation probably never will be. Single user only.
|
||||
* Bundle mechanism works for basic usage, but is extremely ineficient and **does not do anything with blocked identities**.
|
||||
* Not thread safe, never will be. Single user only.
|
||||
* Bundle works, but is inefficient. Will optimize after proof of concept.
|
||||
|
||||
# Installation
|
||||
|
||||
|
@ -85,11 +86,11 @@ TODO
|
|||
- [X] Make CLI names consistent with API names. Eg: find vs. read.
|
||||
- [X] `find-all` should....find all. Currently finds your messages or maybe peers, but not all.
|
||||
- [X] Add log count to `pigeon-cli status`
|
||||
- [ ] Delete `Draft#put` entirely.
|
||||
- [ ] Check block list before ingesting bundles.
|
||||
- [X] Delete `Draft#put` entirely.
|
||||
- [X] Check block list before ingesting bundles.
|
||||
- [ ] Need a way of importing / exporting a feeds blobs. (see "Bundle Brainstorming" below)
|
||||
- [ ] Update README.md / tutorial.rb (user manual for `Pigeon::Database`).
|
||||
- [ ] Make the switch to LevelDB, RocksDB, [UNQLite](https://unqlite.org/features.html) or similar (currently using Ruby PStore).
|
||||
- [ ] Need a way of importing / exporting a feeds blobs. (see "Bundle Brainstorming" below)
|
||||
- [ ] Need a way of adding peers messages / gossip to bundles. (see "Bundle Brainstorming" below)
|
||||
- [ ] add parsers and validators for all CLI inputs
|
||||
- [ ] Reduce whole darn repo into single module to aide portability. `::Helpers` module is OK.
|
||||
|
@ -104,7 +105,8 @@ TODO
|
|||
|
||||
- [ ] (later, not now) Support partial verification via `lipmaa` property.
|
||||
- [ ] Add mandatory `--since=` arg to `bundle create
|
||||
- [ ] Interest and Disinterest Signalling for document routing: Create a `$gossip` message to express `blob.have`, `blob.want` and to note last message received of a peer. This can steer bundle creation and an eventual `--for` flag at bundle creation time to customize a bundle to a particular user.
|
||||
- [ ] Interest and Disinterest Signalling for document routing: Create a `$blob_status` message to express `have`, `want` signalling. This can steer bundle creation and an eventual `--for` flag at bundle creation time to customize a bundle to a particular user.
|
||||
- [ ] Add a schema for `$peer_status`. Eg: `block`, `unblock`, `follow`, `unfollow`.
|
||||
|
||||
# Idea Bin
|
||||
|
||||
|
|
|
@ -187,7 +187,7 @@ module Pigeon
|
|||
author = msg.author
|
||||
signature = msg.signature
|
||||
|
||||
return db.read_message(msg_hash) if db.message?(msg_hash)
|
||||
return db.read_message(msg_hash) if db.message_saved?(msg_hash)
|
||||
|
||||
if key_count > 64
|
||||
msg = MSG_SIZE_ERROR % key_count
|
||||
|
@ -203,7 +203,6 @@ module Pigeon
|
|||
assert("prev", msg.prev, expected_prev)
|
||||
tpl = msg.template.render_without_signature
|
||||
Helpers.verify_string(author, signature, tpl)
|
||||
msg.untaint
|
||||
msg.freeze
|
||||
msg
|
||||
end
|
||||
|
|
|
@ -9,14 +9,15 @@ module Pigeon
|
|||
|
||||
# === PEERS
|
||||
def add_peer(p); store.add_peer(p); end
|
||||
def all_blocks(); store.all_blocks(); end
|
||||
def all_peers(); store.all_peers(); end
|
||||
def block_peer(p); store.block_peer(p); end
|
||||
def remove_peer(p); store.remove_peer(p); end
|
||||
def peer_blocked?(p); store.peer_blocked?(p); end
|
||||
def all_blocks(); store.all_blocks(); end
|
||||
def all_peers(); store.all_peers(); end
|
||||
|
||||
# === MESSAGES
|
||||
def find_all_messages(mhash = nil); store.find_all_messages(mhash); end
|
||||
def message?(multihash); store.message?(multihash); end
|
||||
def message_saved?(multihash); store.message_saved?(multihash); end
|
||||
|
||||
def save_message(msg_obj)
|
||||
store.insert_message(Helpers.verify_message(self, msg_obj))
|
||||
|
|
160
lib/pigeon/helpers.rb
Normal file
160
lib/pigeon/helpers.rb
Normal file
|
@ -0,0 +1,160 @@
|
|||
module Pigeon
|
||||
class Helpers
|
||||
class VerificationError < StandardError; end
|
||||
class MessageSizeError < StandardError; end
|
||||
|
||||
B32_ENC = {
|
||||
"00000" => "0", "00001" => "1", "00010" => "2", "00011" => "3",
|
||||
"00100" => "4", "00101" => "5", "00110" => "6", "00111" => "7",
|
||||
"01000" => "8", "01001" => "9", "01010" => "A", "01011" => "B",
|
||||
"01100" => "C", "01101" => "D", "01110" => "E", "01111" => "F",
|
||||
"10000" => "G", "10001" => "H", "10010" => "J", "10011" => "K",
|
||||
"10100" => "M", "10101" => "N", "10110" => "P", "10111" => "Q",
|
||||
"11000" => "R", "11001" => "S", "11010" => "T", "11011" => "V",
|
||||
"11100" => "W", "11101" => "X", "11110" => "Y", "11111" => "Z",
|
||||
}.freeze
|
||||
|
||||
B32_DEC = {
|
||||
"0" => 0b00000, "O" => 0b00000, "1" => 0b00001, "I" => 0b00001,
|
||||
"L" => 0b00001, "2" => 0b00010, "3" => 0b00011, "4" => 0b00100,
|
||||
"5" => 0b00101, "6" => 0b00110, "7" => 0b00111, "8" => 0b01000,
|
||||
"9" => 0b01001, "A" => 0b01010, "B" => 0b01011, "C" => 0b01100,
|
||||
"D" => 0b01101, "E" => 0b01110, "F" => 0b01111, "G" => 0b10000,
|
||||
"H" => 0b10001, "J" => 0b10010, "K" => 0b10011, "M" => 0b10100,
|
||||
"N" => 0b10101, "P" => 0b10110, "Q" => 0b10111, "R" => 0b11000,
|
||||
"S" => 0b11001, "T" => 0b11010, "V" => 0b11011, "W" => 0b11100,
|
||||
"X" => 0b11101, "Y" => 0b11110, "Z" => 0b11111,
|
||||
}.freeze
|
||||
|
||||
def self.lipmaa(n)
|
||||
# The original lipmaa function returns -1 for 0
|
||||
# but that does not mesh well with our serialization
|
||||
# scheme. Comments welcome on this one.
|
||||
return 0 if n < 1 # Prevent -1, division by zero etc..
|
||||
|
||||
m, po3, x = 1, 3, n
|
||||
# find k such that (3^k - 1)/2 >= n
|
||||
while (m < n)
|
||||
po3 *= 3
|
||||
m = (po3 - 1) / 2
|
||||
end
|
||||
po3 /= 3
|
||||
# find longest possible backjump
|
||||
if (m != n)
|
||||
while x != 0
|
||||
m = (po3 - 1) / 2
|
||||
po3 /= 3
|
||||
x %= m
|
||||
end
|
||||
if (m != po3)
|
||||
po3 = m
|
||||
end
|
||||
end
|
||||
return n - po3
|
||||
end
|
||||
|
||||
# http://www.crockford.com/wrmg/base32.html
|
||||
def self.b32_encode(string)
|
||||
string
|
||||
.each_byte
|
||||
.to_a
|
||||
.map { |x| x.to_s(2).rjust(8, "0") }
|
||||
.join
|
||||
.scan(/.{1,5}/)
|
||||
.map { |x| x.ljust(5, "0") }
|
||||
.map { |bits| B32_ENC.fetch(bits) }
|
||||
.join
|
||||
end
|
||||
|
||||
# http://www.crockford.com/wrmg/base32.html
|
||||
def self.b32_decode(string)
|
||||
string
|
||||
.split("")
|
||||
.map { |x| B32_DEC.fetch(x.upcase) }
|
||||
.map { |x| x.to_s(2).rjust(5, "0") }
|
||||
.join("")
|
||||
.scan(/.{1,8}/)
|
||||
.map { |x| x.length == 8 ? x.to_i(2).chr : "" }
|
||||
.join("")
|
||||
end
|
||||
|
||||
def self.verify_string(identity, string_signature, string)
|
||||
binary_signature = decode_multihash(string_signature)
|
||||
|
||||
string_key = identity.multihash
|
||||
binary_key = decode_multihash(string_key)
|
||||
verify_key = Ed25519::VerifyKey.new(binary_key)
|
||||
|
||||
verify_key.verify(binary_signature, string)
|
||||
end
|
||||
|
||||
def self.assert(field, actual, expected)
|
||||
unless actual == expected
|
||||
message = VERFIY_ERROR % [field, actual || "nil", expected || "nil"]
|
||||
raise VerificationError, message
|
||||
end
|
||||
end
|
||||
|
||||
def self.publish_draft(db, draft)
|
||||
author = db.local_identity
|
||||
mhash = author.multihash
|
||||
template = MessageSerializer.new(draft)
|
||||
depth = db.get_message_count_for(mhash)
|
||||
|
||||
draft.author = author
|
||||
draft.depth = depth
|
||||
draft.prev = db.get_message_by_depth(mhash, depth - 1)
|
||||
draft.lipmaa = Helpers.lipmaa(depth)
|
||||
|
||||
unsigned = template.render_without_signature
|
||||
draft.signature = author.sign(unsigned)
|
||||
tokens = Lexer.tokenize_unsigned(unsigned, draft.signature)
|
||||
message = Parser.parse(db, tokens)[0]
|
||||
db.reset_draft
|
||||
message
|
||||
end
|
||||
|
||||
def self.update_draft(db, key, value)
|
||||
draft = db.current_draft
|
||||
draft[key] = value
|
||||
db.save_draft(draft)
|
||||
return draft.body[key]
|
||||
end
|
||||
|
||||
def self.verify_message(db, msg)
|
||||
msg_hash = msg.multihash
|
||||
body = msg.body
|
||||
key_count = body.count
|
||||
author = msg.author
|
||||
signature = msg.signature
|
||||
|
||||
return db.read_message(msg_hash) if db.message_saved?(msg_hash)
|
||||
|
||||
if key_count > 64
|
||||
msg = MSG_SIZE_ERROR % key_count
|
||||
raise MessageSizeError, msg
|
||||
end
|
||||
count = db.get_message_count_for(author.multihash)
|
||||
expected_prev = db.get_message_by_depth(author.multihash, count - 1) || Pigeon::NOTHING
|
||||
assert("depth", count, msg.depth)
|
||||
# TODO: Re-visit this. Our current verification method
|
||||
# is probably too strict and won't allow for partial
|
||||
# verification of feeds.
|
||||
assert("lipmaa", Helpers.lipmaa(msg.depth), msg.lipmaa)
|
||||
assert("prev", msg.prev, expected_prev)
|
||||
tpl = msg.template.render_without_signature
|
||||
Helpers.verify_string(author, signature, tpl)
|
||||
msg.untaint
|
||||
msg.freeze
|
||||
msg
|
||||
end
|
||||
|
||||
def self.decode_multihash(string)
|
||||
if string[SIG_RANGE] == SIG_FOOTER
|
||||
return b32_decode(string.gsub(SIG_FOOTER, ""))
|
||||
else
|
||||
return b32_decode(string[1, -1].gsub(FOOTERS_REGEX, ""))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -40,7 +40,9 @@ module Pigeon
|
|||
|
||||
def finish_this_message!
|
||||
@scratchpad.freeze
|
||||
@results.push(@db.ingest_message(**@scratchpad))
|
||||
unless @db.peer_blocked?(@scratchpad.fetch(:author))
|
||||
@results.push(@db.ingest_message(**@scratchpad))
|
||||
end
|
||||
reset_scratchpad
|
||||
end
|
||||
|
||||
|
|
|
@ -97,25 +97,34 @@ module Pigeon
|
|||
read { store[MESSAGE_BY_DEPTH_NS][key] }
|
||||
end
|
||||
|
||||
def message?(multihash)
|
||||
read { store[MESG_NS].fetch(multihash, false) }
|
||||
end
|
||||
|
||||
def read_message(multihash)
|
||||
read { store[MESG_NS].fetch(multihash) }
|
||||
end
|
||||
|
||||
def insert_message(msg)
|
||||
if msg.tainted?
|
||||
STDERR.puts "WARNING: Just saved an unverified message"
|
||||
end
|
||||
write do
|
||||
return msg if store[MESG_NS][msg.multihash]
|
||||
if store[MESG_NS].fetch(msg.multihash, false)
|
||||
return msg
|
||||
end
|
||||
|
||||
if store[BLCK_NS].member?(msg.author.multihash)
|
||||
STDERR.puts("Blocked peer: #{msg.author.multihash}")
|
||||
return msg
|
||||
end
|
||||
|
||||
insert_and_update_index(msg)
|
||||
msg
|
||||
end
|
||||
end
|
||||
|
||||
def message_saved?(multihash)
|
||||
read { store[MESG_NS].fetch(multihash, false) }
|
||||
end
|
||||
|
||||
def peer_blocked?(multihash)
|
||||
read { store[BLCK_NS].member?(multihash) }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def bootstrap
|
||||
|
|
9
spec/fixtures/x.bundle
vendored
Normal file
9
spec/fixtures/x.bundle
vendored
Normal file
|
@ -0,0 +1,9 @@
|
|||
author @PPJQ3Q36W258VQ1NKYY2G7VW24J8NMAACHXCD83GCQ3K8F4C9X2G.ed25519
|
||||
kind theSlowestWayToSendAMessageEver
|
||||
prev %K2EKKRFA5F34VG9JSW17SS5HNQAKZTTNP6A9DVR2ZBPG8GME3QVG.sha256
|
||||
depth 2
|
||||
lipmaa 1
|
||||
|
||||
yo_dawg:"i heard you like gossip protocols"
|
||||
|
||||
signature QM8ERCNP4NM4Y3PY09EWVCWJ6MJKPRN67EX4P8MN6DE47BZSG4M3XXCT3JWVRR7D0232Z5HYCQ5J8744JNFBYXFN8JGXM0R0ZESSR2R.sig.ed25519
|
|
@ -31,4 +31,12 @@ RSpec.describe Pigeon::Message do
|
|||
db.create_bundle
|
||||
db.ingest_bundle
|
||||
end
|
||||
|
||||
it "does not ingest messages from blocked peers" do
|
||||
db.reset_database
|
||||
antagonist = "@PPJQ3Q36W258VQ1NKYY2G7VW24J8NMAACHXCD83GCQ3K8F4C9X2G.ed25519"
|
||||
db.block_peer(antagonist)
|
||||
db.ingest_bundle("./spec/fixtures/x.bundle")
|
||||
expect(db.find_all_messages.count).to eq(0)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -10,7 +10,7 @@ db.publish_draft
|
|||
db.save_draft
|
||||
db.save_message
|
||||
db.reset_current_draft
|
||||
db.message?
|
||||
db.message_saved?
|
||||
db.read_message
|
||||
db.create_message
|
||||
db.find_all_messages
|
||||
|
|
30
x.bundle
30
x.bundle
|
@ -1,30 +0,0 @@
|
|||
author @PPJQ3Q36W258VQ1NKYY2G7VW24J8NMAACHXCD83GCQ3K8F4C9X2G.ed25519
|
||||
kind carlinoType
|
||||
prev NONE
|
||||
depth 0
|
||||
lipmaa 0
|
||||
|
||||
author:"daniel"
|
||||
company:"Dev Man Dan LLC"
|
||||
|
||||
signature 2H7WM4F53YDJ2CR6XMTM6S7HSDMHD5K4Y9A2DYXWDDSRBZB1KN092FC6H69RB79VQVZ769NY5VRT7FXJVN2B2DJ163H5ZWSYV3WVY2G.sig.ed25519
|
||||
|
||||
author @PPJQ3Q36W258VQ1NKYY2G7VW24J8NMAACHXCD83GCQ3K8F4C9X2G.ed25519
|
||||
kind MESSAGE_TYPE_HERE
|
||||
prev %N4QCWKV4ERKKV5BHBYAQK0AX1VQDAMFX972SK1E4BR3HDHXMFPVG.sha256
|
||||
depth 1
|
||||
lipmaa 0
|
||||
|
||||
my_key:"my_value"
|
||||
|
||||
signature JD8HGEMTQKCNV7RM488VAV1XC54W4MT2Y0G4N2VME4G4QFXT2H670M3YMY4W6E1R3MTGJBADGYGW2DJPT14R9TWM87SMNNNZ6Q2NE00.sig.ed25519
|
||||
|
||||
author @PPJQ3Q36W258VQ1NKYY2G7VW24J8NMAACHXCD83GCQ3K8F4C9X2G.ed25519
|
||||
kind theSlowestWayToSendAMessageEver
|
||||
prev %K2EKKRFA5F34VG9JSW17SS5HNQAKZTTNP6A9DVR2ZBPG8GME3QVG.sha256
|
||||
depth 2
|
||||
lipmaa 1
|
||||
|
||||
yo_dawg:"i heard you like gossip protocols"
|
||||
|
||||
signature QM8ERCNP4NM4Y3PY09EWVCWJ6MJKPRN67EX4P8MN6DE47BZSG4M3XXCT3JWVRR7D0232Z5HYCQ5J8744JNFBYXFN8JGXM0R0ZESSR2R.sig.ed25519
|
Loading…
Reference in New Issue
Block a user