Continue implementing and test bundle imports

README.md

@@ -15,24 +15,28 @@ You can override this value by specifying a `PIGEON_PATH` ENV var.
 
 Want to get involved? Below are a few things I need help with.
 
-Email `contact@vaporsfot.xyz` if you have any questions.
+Email `contact@vaporsoft.xyz` if you have any questions.
 
  * Writing a BNF grammar for message parsing
  * Test coverage increases
  * Manual QA of features and edge cases
  * Providing constructive feedback on documentation
  * Cross-compiling windows binaries
- * General Golang help (I am a Golang novice- project structure could be improved)
  * Security auditing and vulnerability discovery. Please send security concerns to `contact@vaporsoft.xyz` privately.
 
 # TODO
 
  - [ ] Add a real testing lib to DRY things up.
+ - [ ] Validate and scrutinize `depth`, `lipmaa`, `prev` fields when ingesting message bundles to account for poorly written peer clients.
  - [ ] Get a good CI system going? Run tests at PR time, provide prebuilt binaries, prevent coverage slips, etc..
  - [ ] Add a `transact()` helper to ensure all transactions are closed out.
  - [ ] Switch to [SQLX](https://github.com/jmoiron/sqlx) for extra sanity.
  - [ ] Write docs for all CLI commands / args AFTER completion.
- - [ ] Finish all the things below:
+ - [ ] Start using the `check` helper instead of `error != nil`.
+ - [ ] Update spec to only allow UPPERCASE MULTIHASHES
+ - [ ] Implement `query.pgn` protocol, as outlined [here](%CSBzyskUxqbFSgOBh8OkVLn18NqX3zu3CF58mm2JHok=.sha256) and [here](%KWETmo1cmlfYK4N6FVL9BHYfFcKMy49E94XGuZSPGCw=.sha256).
+ - [ ] Add a note about "shallow" vs. "deep" verification.
+ - [ ] Finish all the things below
 
 # Protocol Changes?
 

project/cli.go

@@ -96,7 +96,7 @@ var peerFollowCmd = &cobra.Command{
 	},
 }
 
-var peerUntrackedCmd = &cobra.Command{
+var peerUntrackCmd = &cobra.Command{
 	Use:     "untrack",
 	Short:   "Stop following/blocking a peer",
 	Aliases: []string{"unblock", "unfollow"},
@@ -171,7 +171,7 @@ func BootstrapCLI() {
 	rootCmd.AddCommand(peerRootCmd)
 	peerRootCmd.AddCommand(peerBlockCmd)
 	peerRootCmd.AddCommand(peerFollowCmd)
-	peerRootCmd.AddCommand(peerUntrackedCmd)
+	peerRootCmd.AddCommand(peerUntrackCmd)
 	peerRootCmd.AddCommand(peerListCmd)
 
 	rootCmd.AddCommand(blobRootCmd)

project/decoders.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"strings"
 )
 
@@ -14,10 +13,7 @@ type testCase struct {
 // to a byte array.
 func B32Decode(input string) []byte {
 	output, error := encoder.DecodeString(input)
-	if error != nil {
-		panic(fmt.Sprintf("Error decoding Base32 string %s", input))
-	}
-
+	check(error, "Error decoding Base32 string %s", input)
 	return output
 }
 

project/import_bundle.go

@@ -6,23 +6,28 @@ import (
 	"io/ioutil"
 )
 
-/** ingestRelevantMessages takes an array of Pigeon messages
+func ingestOneMessage(msg pigeonMessage, blobIndex map[string]bool) {
+	if getPeerStatus(msg.author) == following {
+		fmt.Println("TODO: Ingest this message")
+	}
+}
+
+/** ingestManyMessages takes an array of Pigeon messages
 and adds them to the local database, assuming that they are
 messages of interest. */
-func ingestRelevantMessages(msgs []pigeonMessage) {
-	for _, message := range msgs {
-		fmt.Printf("Peer %s has %s status\n", message.author[0:13], getPeerStatus(message.author))
+func ingestManyMessages(outp parserOutput) {
+	for _, message := range outp.messages {
+		ingestOneMessage(message, outp.blobIndex)
 	}
-	panic("This is where I stopped")
 }
 
 func importBundle(path string) error {
 	// Get messages.pgn file
 	dat, err1 := ioutil.ReadFile(path)
-	check(err1, "Problem opening %s. Error: %s", path, err1)
-	msgs, err2 := parseMessage(string(dat))
+	check(err1, "Problem opening bundle %s. Error: %s", path, err1)
+	outp, err2 := parseMessage(string(dat))
 	check(err2, "Failed to parse %s. Error: %s", path, err2)
-	ingestRelevantMessages(msgs)
+	ingestManyMessages(outp)
 	// Parse messages
 	// Map over messages
 	return errors.New("Not done yet")

project/import_bundle_test.go

@@ -1,10 +1,21 @@
 package main
 
-import "testing"
+import (
+	"fmt"
+	"testing"
+)
 
 func TestImportBundle(t *testing.T) {
+	author := "USER.09XBQDDGZPEKFBFBY67XNR5QA0TRWAKYKYNEDNQTZJV0F1JB0DGG"
+	addPeer(author, following)
 	error := importBundle("../fixtures/has_blobs/messages.pgn")
-	if error != nil {
-		t.Fatalf("Error while importing: %s", error)
-	}
+	check(error, "Error while importing: %s", error)
+
+	fmt.Println("NEXT STEP: Assert that we have the following assets:")
+	fmt.Println("FILE.622PRNJ7C0S05XR2AHDPKWMG051B1QW5SXMN2RQHF2AND6J8VGPG")
+	fmt.Println("FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG")
+	fmt.Println("FILE.YPF11E5N9JFVB6KB1N1WDVVT9DXMCHE0XJWBZHT2CQ29S5SEPCSG")
+	fmt.Println("TEXT.RGKRHC0APNN9FCJTVBN1NR1ZYQ9ZY34PYYASSMJ6016S30ZTWHR0")
+	fmt.Println("TEXT.V52B1GH1XS8K1QKJG3AK127XYA23E82J0A2ZQTJ08TF8NZN2A1Y0")
+	fmt.Println("TEXT.Z3QS1HPX756E22XWKXAXH7NTSTJGY0AHEM9KQNATTC6HHCACZGN0")
 }

project/parser.go

@@ -39,6 +39,17 @@ type parserState struct {
 	error   error
 }
 
+type parserOutput struct {
+	/** `messages` is an array of messages. The messages are SHALLOW
+	verified. That means the message has a valid signature and syntax,
+	but `depth`, `lipmaa` and `prev` have not been scrutinized for validity. */
+	messages []pigeonMessage
+	/** `blobIndex` is a hash where keys represent each unique blob
+	foud in a bundle. This is required to avoid ingesting unwanted
+	blobs to disk. */
+	blobIndex map[string]bool
+}
+
 func newState(message string) parserState {
 	return parserState{
 		mode:    parsingHeader,
@@ -46,12 +57,16 @@ func newState(message string) parserState {
 	}
 }
 
-func parseMessage(message string) ([]pigeonMessage, error) {
+func parseMessage(message string) (parserOutput, error) {
+	empty := parserOutput{
+		messages:  []pigeonMessage{},
+		blobIndex: map[string]bool{},
+	}
 	state := newState(message)
 	for state.scanner.Scan() {
 		// Exit early if any step produces an error.
 		if state.error != nil {
-			return []pigeonMessage{}, state.error
+			return empty, state.error
 		}
 
 		switch state.mode {
@@ -68,9 +83,16 @@ func parseMessage(message string) ([]pigeonMessage, error) {
 		}
 	}
 	if state.mode == parsingError {
-		return []pigeonMessage{}, state.error
+		return empty, state.error
 	}
-	return state.results, nil
+	blobIndex := map[string]bool{}
+	for _, msg := range state.results {
+		for _, pair := range msg.body {
+			panicf("YOU NEED TO FINISH CREATING A BLOB INDEX FOR IMPORTED BUNDLES: %s", pair.key)
+		}
+	}
+	output := parserOutput{messages: state.results, blobIndex: blobIndex}
+	return output, nil
 }
 
 func parseHeader(state *parserState) {

project/parser_test.go

@@ -18,7 +18,7 @@ func TestParser(t *testing.T) {
 	}
 
 	fixtureSize := 13
-	length := len(output)
+	length := len(output.messages)
 	if length != fixtureSize {
 		t.Fatalf("Expected %d items, got %d", fixtureSize, length)
 	}
@@ -29,8 +29,8 @@ func TestParser2(t *testing.T) {
 	if err1 != nil {
 		log.Fatal(err1)
 	}
-	output, err2 := parseMessage(string(content))
-
+	parserOutput, err2 := parseMessage(string(content))
+	output := parserOutput.messages
 	if err2 != nil {
 		log.Fatal(err2)
 	}