From 6face734823956f8348d808d49e6b9bd489d0f46 Mon Sep 17 00:00:00 2001 From: Rick Carlino Date: Wed, 28 Oct 2020 08:06:56 -0500 Subject: [PATCH] Continue implementing and test bundle imports --- README.md | 10 +++++++--- project/cli.go | 4 ++-- project/decoders.go | 6 +----- project/import_bundle.go | 21 +++++++++++++-------- project/import_bundle_test.go | 19 +++++++++++++++---- project/parser.go | 30 ++++++++++++++++++++++++++---- project/parser_test.go | 6 +++--- 7 files changed, 67 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 85f057d..96570bf 100644 --- a/README.md +++ b/README.md @@ -15,24 +15,28 @@ You can override this value by specifying a `PIGEON_PATH` ENV var. Want to get involved? Below are a few things I need help with. -Email `contact@vaporsfot.xyz` if you have any questions. +Email `contact@vaporsoft.xyz` if you have any questions. * Writing a BNF grammar for message parsing * Test coverage increases * Manual QA of features and edge cases * Providing constructive feedback on documentation * Cross-compiling windows binaries - * General Golang help (I am a Golang novice- project structure could be improved) * Security auditing and vulnerability discovery. Please send security concerns to `contact@vaporsoft.xyz` privately. # TODO - [ ] Add a real testing lib to DRY things up. + - [ ] Validate and scrutinize `depth`, `lipmaa`, `prev` fields when ingesting message bundles to account for poorly written peer clients. - [ ] Get a good CI system going? Run tests at PR time, provide prebuilt binaries, prevent coverage slips, etc.. - [ ] Add a `transact()` helper to ensure all transactions are closed out. - [ ] Switch to [SQLX](https://github.com/jmoiron/sqlx) for extra sanity. - [ ] Write docs for all CLI commands / args AFTER completion. - - [ ] Finish all the things below: + - [ ] Start using the `check` helper instead of `error != nil`. + - [ ] Update spec to only allow UPPERCASE MULTIHASHES + - [ ] Implement `query.pgn` protocol, as outlined [here](%CSBzyskUxqbFSgOBh8OkVLn18NqX3zu3CF58mm2JHok=.sha256) and [here](%KWETmo1cmlfYK4N6FVL9BHYfFcKMy49E94XGuZSPGCw=.sha256). + - [ ] Add a note about "shallow" vs. "deep" verification. + - [ ] Finish all the things below # Protocol Changes? diff --git a/project/cli.go b/project/cli.go index a304acc..259c666 100644 --- a/project/cli.go +++ b/project/cli.go @@ -96,7 +96,7 @@ var peerFollowCmd = &cobra.Command{ }, } -var peerUntrackedCmd = &cobra.Command{ +var peerUntrackCmd = &cobra.Command{ Use: "untrack", Short: "Stop following/blocking a peer", Aliases: []string{"unblock", "unfollow"}, @@ -171,7 +171,7 @@ func BootstrapCLI() { rootCmd.AddCommand(peerRootCmd) peerRootCmd.AddCommand(peerBlockCmd) peerRootCmd.AddCommand(peerFollowCmd) - peerRootCmd.AddCommand(peerUntrackedCmd) + peerRootCmd.AddCommand(peerUntrackCmd) peerRootCmd.AddCommand(peerListCmd) rootCmd.AddCommand(blobRootCmd) diff --git a/project/decoders.go b/project/decoders.go index 1dae61c..d8e7c1a 100644 --- a/project/decoders.go +++ b/project/decoders.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "strings" ) @@ -14,10 +13,7 @@ type testCase struct { // to a byte array. func B32Decode(input string) []byte { output, error := encoder.DecodeString(input) - if error != nil { - panic(fmt.Sprintf("Error decoding Base32 string %s", input)) - } - + check(error, "Error decoding Base32 string %s", input) return output } diff --git a/project/import_bundle.go b/project/import_bundle.go index 4c288c2..b652fa9 100644 --- a/project/import_bundle.go +++ b/project/import_bundle.go @@ -6,23 +6,28 @@ import ( "io/ioutil" ) -/** ingestRelevantMessages takes an array of Pigeon messages +func ingestOneMessage(msg pigeonMessage, blobIndex map[string]bool) { + if getPeerStatus(msg.author) == following { + fmt.Println("TODO: Ingest this message") + } +} + +/** ingestManyMessages takes an array of Pigeon messages and adds them to the local database, assuming that they are messages of interest. */ -func ingestRelevantMessages(msgs []pigeonMessage) { - for _, message := range msgs { - fmt.Printf("Peer %s has %s status\n", message.author[0:13], getPeerStatus(message.author)) +func ingestManyMessages(outp parserOutput) { + for _, message := range outp.messages { + ingestOneMessage(message, outp.blobIndex) } - panic("This is where I stopped") } func importBundle(path string) error { // Get messages.pgn file dat, err1 := ioutil.ReadFile(path) - check(err1, "Problem opening %s. Error: %s", path, err1) - msgs, err2 := parseMessage(string(dat)) + check(err1, "Problem opening bundle %s. Error: %s", path, err1) + outp, err2 := parseMessage(string(dat)) check(err2, "Failed to parse %s. Error: %s", path, err2) - ingestRelevantMessages(msgs) + ingestManyMessages(outp) // Parse messages // Map over messages return errors.New("Not done yet") diff --git a/project/import_bundle_test.go b/project/import_bundle_test.go index e4668ba..b9b108d 100644 --- a/project/import_bundle_test.go +++ b/project/import_bundle_test.go @@ -1,10 +1,21 @@ package main -import "testing" +import ( + "fmt" + "testing" +) func TestImportBundle(t *testing.T) { + author := "USER.09XBQDDGZPEKFBFBY67XNR5QA0TRWAKYKYNEDNQTZJV0F1JB0DGG" + addPeer(author, following) error := importBundle("../fixtures/has_blobs/messages.pgn") - if error != nil { - t.Fatalf("Error while importing: %s", error) - } + check(error, "Error while importing: %s", error) + + fmt.Println("NEXT STEP: Assert that we have the following assets:") + fmt.Println("FILE.622PRNJ7C0S05XR2AHDPKWMG051B1QW5SXMN2RQHF2AND6J8VGPG") + fmt.Println("FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG") + fmt.Println("FILE.YPF11E5N9JFVB6KB1N1WDVVT9DXMCHE0XJWBZHT2CQ29S5SEPCSG") + fmt.Println("TEXT.RGKRHC0APNN9FCJTVBN1NR1ZYQ9ZY34PYYASSMJ6016S30ZTWHR0") + fmt.Println("TEXT.V52B1GH1XS8K1QKJG3AK127XYA23E82J0A2ZQTJ08TF8NZN2A1Y0") + fmt.Println("TEXT.Z3QS1HPX756E22XWKXAXH7NTSTJGY0AHEM9KQNATTC6HHCACZGN0") } diff --git a/project/parser.go b/project/parser.go index c6285af..7161079 100644 --- a/project/parser.go +++ b/project/parser.go @@ -39,6 +39,17 @@ type parserState struct { error error } +type parserOutput struct { + /** `messages` is an array of messages. The messages are SHALLOW + verified. That means the message has a valid signature and syntax, + but `depth`, `lipmaa` and `prev` have not been scrutinized for validity. */ + messages []pigeonMessage + /** `blobIndex` is a hash where keys represent each unique blob + foud in a bundle. This is required to avoid ingesting unwanted + blobs to disk. */ + blobIndex map[string]bool +} + func newState(message string) parserState { return parserState{ mode: parsingHeader, @@ -46,12 +57,16 @@ func newState(message string) parserState { } } -func parseMessage(message string) ([]pigeonMessage, error) { +func parseMessage(message string) (parserOutput, error) { + empty := parserOutput{ + messages: []pigeonMessage{}, + blobIndex: map[string]bool{}, + } state := newState(message) for state.scanner.Scan() { // Exit early if any step produces an error. if state.error != nil { - return []pigeonMessage{}, state.error + return empty, state.error } switch state.mode { @@ -68,9 +83,16 @@ func parseMessage(message string) ([]pigeonMessage, error) { } } if state.mode == parsingError { - return []pigeonMessage{}, state.error + return empty, state.error } - return state.results, nil + blobIndex := map[string]bool{} + for _, msg := range state.results { + for _, pair := range msg.body { + panicf("YOU NEED TO FINISH CREATING A BLOB INDEX FOR IMPORTED BUNDLES: %s", pair.key) + } + } + output := parserOutput{messages: state.results, blobIndex: blobIndex} + return output, nil } func parseHeader(state *parserState) { diff --git a/project/parser_test.go b/project/parser_test.go index bd71ec4..bb0747f 100644 --- a/project/parser_test.go +++ b/project/parser_test.go @@ -18,7 +18,7 @@ func TestParser(t *testing.T) { } fixtureSize := 13 - length := len(output) + length := len(output.messages) if length != fixtureSize { t.Fatalf("Expected %d items, got %d", fixtureSize, length) } @@ -29,8 +29,8 @@ func TestParser2(t *testing.T) { if err1 != nil { log.Fatal(err1) } - output, err2 := parseMessage(string(content)) - + parserOutput, err2 := parseMessage(string(content)) + output := parserOutput.messages if err2 != nil { log.Fatal(err2) }