Continue implementing and test bundle imports
This commit is contained in:
parent
1f4aa88263
commit
6face73482
10
README.md
10
README.md
|
@ -15,24 +15,28 @@ You can override this value by specifying a `PIGEON_PATH` ENV var.
|
|||
|
||||
Want to get involved? Below are a few things I need help with.
|
||||
|
||||
Email `contact@vaporsfot.xyz` if you have any questions.
|
||||
Email `contact@vaporsoft.xyz` if you have any questions.
|
||||
|
||||
* Writing a BNF grammar for message parsing
|
||||
* Test coverage increases
|
||||
* Manual QA of features and edge cases
|
||||
* Providing constructive feedback on documentation
|
||||
* Cross-compiling windows binaries
|
||||
* General Golang help (I am a Golang novice- project structure could be improved)
|
||||
* Security auditing and vulnerability discovery. Please send security concerns to `contact@vaporsoft.xyz` privately.
|
||||
|
||||
# TODO
|
||||
|
||||
- [ ] Add a real testing lib to DRY things up.
|
||||
- [ ] Validate and scrutinize `depth`, `lipmaa`, `prev` fields when ingesting message bundles to account for poorly written peer clients.
|
||||
- [ ] Get a good CI system going? Run tests at PR time, provide prebuilt binaries, prevent coverage slips, etc..
|
||||
- [ ] Add a `transact()` helper to ensure all transactions are closed out.
|
||||
- [ ] Switch to [SQLX](https://github.com/jmoiron/sqlx) for extra sanity.
|
||||
- [ ] Write docs for all CLI commands / args AFTER completion.
|
||||
- [ ] Finish all the things below:
|
||||
- [ ] Start using the `check` helper instead of `error != nil`.
|
||||
- [ ] Update spec to only allow UPPERCASE MULTIHASHES
|
||||
- [ ] Implement `query.pgn` protocol, as outlined [here](%CSBzyskUxqbFSgOBh8OkVLn18NqX3zu3CF58mm2JHok=.sha256) and [here](%KWETmo1cmlfYK4N6FVL9BHYfFcKMy49E94XGuZSPGCw=.sha256).
|
||||
- [ ] Add a note about "shallow" vs. "deep" verification.
|
||||
- [ ] Finish all the things below
|
||||
|
||||
# Protocol Changes?
|
||||
|
||||
|
|
|
@ -96,7 +96,7 @@ var peerFollowCmd = &cobra.Command{
|
|||
},
|
||||
}
|
||||
|
||||
var peerUntrackedCmd = &cobra.Command{
|
||||
var peerUntrackCmd = &cobra.Command{
|
||||
Use: "untrack",
|
||||
Short: "Stop following/blocking a peer",
|
||||
Aliases: []string{"unblock", "unfollow"},
|
||||
|
@ -171,7 +171,7 @@ func BootstrapCLI() {
|
|||
rootCmd.AddCommand(peerRootCmd)
|
||||
peerRootCmd.AddCommand(peerBlockCmd)
|
||||
peerRootCmd.AddCommand(peerFollowCmd)
|
||||
peerRootCmd.AddCommand(peerUntrackedCmd)
|
||||
peerRootCmd.AddCommand(peerUntrackCmd)
|
||||
peerRootCmd.AddCommand(peerListCmd)
|
||||
|
||||
rootCmd.AddCommand(blobRootCmd)
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
@ -14,10 +13,7 @@ type testCase struct {
|
|||
// to a byte array.
|
||||
func B32Decode(input string) []byte {
|
||||
output, error := encoder.DecodeString(input)
|
||||
if error != nil {
|
||||
panic(fmt.Sprintf("Error decoding Base32 string %s", input))
|
||||
}
|
||||
|
||||
check(error, "Error decoding Base32 string %s", input)
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
|
@ -6,23 +6,28 @@ import (
|
|||
"io/ioutil"
|
||||
)
|
||||
|
||||
/** ingestRelevantMessages takes an array of Pigeon messages
|
||||
func ingestOneMessage(msg pigeonMessage, blobIndex map[string]bool) {
|
||||
if getPeerStatus(msg.author) == following {
|
||||
fmt.Println("TODO: Ingest this message")
|
||||
}
|
||||
}
|
||||
|
||||
/** ingestManyMessages takes an array of Pigeon messages
|
||||
and adds them to the local database, assuming that they are
|
||||
messages of interest. */
|
||||
func ingestRelevantMessages(msgs []pigeonMessage) {
|
||||
for _, message := range msgs {
|
||||
fmt.Printf("Peer %s has %s status\n", message.author[0:13], getPeerStatus(message.author))
|
||||
func ingestManyMessages(outp parserOutput) {
|
||||
for _, message := range outp.messages {
|
||||
ingestOneMessage(message, outp.blobIndex)
|
||||
}
|
||||
panic("This is where I stopped")
|
||||
}
|
||||
|
||||
func importBundle(path string) error {
|
||||
// Get messages.pgn file
|
||||
dat, err1 := ioutil.ReadFile(path)
|
||||
check(err1, "Problem opening %s. Error: %s", path, err1)
|
||||
msgs, err2 := parseMessage(string(dat))
|
||||
check(err1, "Problem opening bundle %s. Error: %s", path, err1)
|
||||
outp, err2 := parseMessage(string(dat))
|
||||
check(err2, "Failed to parse %s. Error: %s", path, err2)
|
||||
ingestRelevantMessages(msgs)
|
||||
ingestManyMessages(outp)
|
||||
// Parse messages
|
||||
// Map over messages
|
||||
return errors.New("Not done yet")
|
||||
|
|
|
@ -1,10 +1,21 @@
|
|||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestImportBundle(t *testing.T) {
|
||||
author := "USER.09XBQDDGZPEKFBFBY67XNR5QA0TRWAKYKYNEDNQTZJV0F1JB0DGG"
|
||||
addPeer(author, following)
|
||||
error := importBundle("../fixtures/has_blobs/messages.pgn")
|
||||
if error != nil {
|
||||
t.Fatalf("Error while importing: %s", error)
|
||||
}
|
||||
check(error, "Error while importing: %s", error)
|
||||
|
||||
fmt.Println("NEXT STEP: Assert that we have the following assets:")
|
||||
fmt.Println("FILE.622PRNJ7C0S05XR2AHDPKWMG051B1QW5SXMN2RQHF2AND6J8VGPG")
|
||||
fmt.Println("FILE.FV0FJ0YZADY7C5JTTFYPKDBHTZJ5JVVP5TCKP0605WWXYJG4VMRG")
|
||||
fmt.Println("FILE.YPF11E5N9JFVB6KB1N1WDVVT9DXMCHE0XJWBZHT2CQ29S5SEPCSG")
|
||||
fmt.Println("TEXT.RGKRHC0APNN9FCJTVBN1NR1ZYQ9ZY34PYYASSMJ6016S30ZTWHR0")
|
||||
fmt.Println("TEXT.V52B1GH1XS8K1QKJG3AK127XYA23E82J0A2ZQTJ08TF8NZN2A1Y0")
|
||||
fmt.Println("TEXT.Z3QS1HPX756E22XWKXAXH7NTSTJGY0AHEM9KQNATTC6HHCACZGN0")
|
||||
}
|
||||
|
|
|
@ -39,6 +39,17 @@ type parserState struct {
|
|||
error error
|
||||
}
|
||||
|
||||
type parserOutput struct {
|
||||
/** `messages` is an array of messages. The messages are SHALLOW
|
||||
verified. That means the message has a valid signature and syntax,
|
||||
but `depth`, `lipmaa` and `prev` have not been scrutinized for validity. */
|
||||
messages []pigeonMessage
|
||||
/** `blobIndex` is a hash where keys represent each unique blob
|
||||
foud in a bundle. This is required to avoid ingesting unwanted
|
||||
blobs to disk. */
|
||||
blobIndex map[string]bool
|
||||
}
|
||||
|
||||
func newState(message string) parserState {
|
||||
return parserState{
|
||||
mode: parsingHeader,
|
||||
|
@ -46,12 +57,16 @@ func newState(message string) parserState {
|
|||
}
|
||||
}
|
||||
|
||||
func parseMessage(message string) ([]pigeonMessage, error) {
|
||||
func parseMessage(message string) (parserOutput, error) {
|
||||
empty := parserOutput{
|
||||
messages: []pigeonMessage{},
|
||||
blobIndex: map[string]bool{},
|
||||
}
|
||||
state := newState(message)
|
||||
for state.scanner.Scan() {
|
||||
// Exit early if any step produces an error.
|
||||
if state.error != nil {
|
||||
return []pigeonMessage{}, state.error
|
||||
return empty, state.error
|
||||
}
|
||||
|
||||
switch state.mode {
|
||||
|
@ -68,9 +83,16 @@ func parseMessage(message string) ([]pigeonMessage, error) {
|
|||
}
|
||||
}
|
||||
if state.mode == parsingError {
|
||||
return []pigeonMessage{}, state.error
|
||||
return empty, state.error
|
||||
}
|
||||
return state.results, nil
|
||||
blobIndex := map[string]bool{}
|
||||
for _, msg := range state.results {
|
||||
for _, pair := range msg.body {
|
||||
panicf("YOU NEED TO FINISH CREATING A BLOB INDEX FOR IMPORTED BUNDLES: %s", pair.key)
|
||||
}
|
||||
}
|
||||
output := parserOutput{messages: state.results, blobIndex: blobIndex}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func parseHeader(state *parserState) {
|
||||
|
|
|
@ -18,7 +18,7 @@ func TestParser(t *testing.T) {
|
|||
}
|
||||
|
||||
fixtureSize := 13
|
||||
length := len(output)
|
||||
length := len(output.messages)
|
||||
if length != fixtureSize {
|
||||
t.Fatalf("Expected %d items, got %d", fixtureSize, length)
|
||||
}
|
||||
|
@ -29,8 +29,8 @@ func TestParser2(t *testing.T) {
|
|||
if err1 != nil {
|
||||
log.Fatal(err1)
|
||||
}
|
||||
output, err2 := parseMessage(string(content))
|
||||
|
||||
parserOutput, err2 := parseMessage(string(content))
|
||||
output := parserOutput.messages
|
||||
if err2 != nil {
|
||||
log.Fatal(err2)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue