Configuration file, arg parsing, database serialization,...

.gitignore

@@ -3,3 +3,4 @@
 *.*~
 *.sw*
 build/*
+usenetsearch.conf

CMakeLists.txt

@@ -18,11 +18,14 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 find_package(OpenSSL REQUIRED)
 
 add_executable(usenetsearch
+    "src/Configuration.cpp"
+    "src/Database.cpp"
     "src/Dns.cpp"
     "src/Except.cpp"
     "src/IoSocket.cpp"
     "src/main.cpp"
     "src/SSLConnection.cpp"
+    "src/StringUtils.cpp"
     "src/TcpConnection.cpp"
     "src/UsenetClient.cpp"
 )

include/usenetsearch/Configuration.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <filesystem>
+#include <string>
+
+#include "usenetsearch/Except.h"
+
+namespace usenetsearch {
+
+struct ConfigurationException: public UsenetSearchException
+{
+    ConfigurationException(int errorCode, const std::string& message):
+        UsenetSearchException(errorCode, message){}
+
+    virtual ~ConfigurationException() = default;
+};
+
+class Configuration
+{
+
+    std::string m_nntpServerHost{"127.0.0.1"};
+    std::string m_nntpServerPassword{"password"};
+    int m_nntpServerPort{119};
+    bool m_nntpServerSSL{false};
+    std::string m_nntpServerUser{"username"};
+    std::filesystem::path m_databasePath{"./db"};
+
+public:
+
+    std::filesystem::path DatabasePath() const;
+    std::string NNTPServerHost() const;
+    std::string NNTPServerPassword() const;
+    int NNTPServerPort() const;
+    bool NNTPServerSSL() const;
+    std::string NNTPServerUser() const;
+    void Open(const std::string& filename);
+
+};
+
+} // namespace usenetsearch

include/usenetsearch/Database.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <vector>
+
+#include "usenetsearch/UsenetClient.h"
+
+namespace usenetsearch {
+
+static constexpr const std::uint64_t DatabaseVersion{1};
+
+class Database{
+
+    std::filesystem::path m_databasePath;
+    std::uint64_t m_databaseVersion{0};
+    std::ifstream m_newsGroupFileInput;
+    std::ofstream m_newsGroupFileOutput;
+
+    void OpenNewsGroupFile();
+
+public:
+
+    ~Database();
+    void Open(std::filesystem::path dbPath);
+    void UpdateNewsgroupList(const std::vector<NntpListEntry>& list);
+
+};
+
+} // namespace usenetsearch

include/usenetsearch/SSLConnection.h

@@ -21,8 +21,6 @@ class SSLConnection : public IoSocket
 {
     enum class SSLReturnState{ RETRY, SUCCESS };
 
-    std::chrono::milliseconds m_connectionTimeout{10000};
-    std::chrono::milliseconds m_ioTimeout{10000};
     std::shared_ptr<SSL> m_ssl;
     std::shared_ptr<SSL_CTX> m_sslContext;
     std::unique_ptr<TcpConnection> m_tcpConnection;
@@ -34,8 +32,8 @@ public:
     SSLConnection(std::unique_ptr<TcpConnection> connection);
     void Connect();
     void Disconnect();
-    std::string Read(size_t amount);
-    void Write(const std::string& data);
+    virtual std::string Read(size_t amount) override;
+    virtual void Write(const std::string& data) override;
 };
 
 } // namespace usenetsearch

include/usenetsearch/StringUtils.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "usenetsearch/Except.h"
+
+namespace usenetsearch {
+
+std::ostream& operator<<(std::ofstream& out, const std::string& str);
+std::ifstream& operator>>(std::ifstream& in, std::string& str);
+std::ostream& operator<<(std::ofstream& out, const std::wstring& str);
+std::ifstream& operator>>(std::ifstream& in, std::wstring& str);
+
+struct StringException: public UsenetSearchException
+{
+    StringException(int errorCode, const std::string& message):
+        UsenetSearchException(errorCode, message){}
+
+    virtual ~StringException() = default;
+};
+
+template<typename T>
+std::vector<T> StringSplit(
+    const T& str,
+    const T& delim,
+    int max_elem = -1
+)
+{
+    size_t pos=0;
+    T s = str;
+    std::vector<T> result;
+    int tokens = 1;
+    while (((pos = s.find(delim)) != T::npos)
+        and ((max_elem < 0) or (tokens != max_elem)))
+    {
+        result.push_back(s.substr(0,pos));
+        s.erase(0, pos + delim.length());
+        tokens++;
+    }
+    if (max_elem != 0) result.push_back(s);
+    return result;
+}
+
+std::string StringLeftTrim(const std::string& str);
+
+std::string StringRightTrim(const std::string& str);
+
+bool StringStartsWith(const std::string& needle, const std::string& haystack);
+
+std::string StringTrim(const std::string& str);
+
+std::string StringToLower(const std::string& str);
+
+bool StringToBoolean(const std::string& str);
+
+} // namespace usenetsearch

include/usenetsearch/TcpConnection.h

@@ -28,8 +28,8 @@ public:
     void Connect(const std::string& host, std::uint16_t port);
     void Disconnect();
     int FileDescriptor() const;
-    std::string Read(size_t amount);
-    void Write(const std::string& data);
+    virtual std::string Read(size_t amount) override;
+    virtual void Write(const std::string& data) override;
 
 };
 

include/usenetsearch/UsenetClient.h

@@ -2,9 +2,11 @@
 
 #include <cstdint>
 #include <codecvt>
+#include <fstream>
 #include <locale>
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "usenetsearch/SSLConnection.h"
 #include "usenetsearch/TcpConnection.h"
@@ -25,6 +27,17 @@ struct NntpMessage
     std::wstring message;
 };
 
+struct NntpListEntry
+{
+    std::wstring name;
+    std::uint64_t high;
+    std::uint64_t low;
+    std::uint64_t count;
+    std::wstring status;
+};
+std::ostream& operator<<(std::ofstream& out, const NntpListEntry& obj);
+std::ifstream& operator>>(std::ifstream& in, NntpListEntry& obj);
+
 class UsenetClient
 {
     std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> m_conv;
@@ -45,6 +58,7 @@ public:
         std::uint16_t port,
         bool useSSL = false
     );
+    std::vector<NntpListEntry> List();
 
 };
 

src/Configuration.cpp

@@ -0,0 +1,122 @@
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+#include "usenetsearch/StringUtils.h"
+
+#include "usenetsearch/Configuration.h"
+
+namespace usenetsearch {
+
+std::filesystem::path Configuration::DatabasePath() const
+{
+    return m_databasePath;
+}
+
+std::string Configuration::NNTPServerHost() const
+{
+    return m_nntpServerHost;
+}
+
+std::string Configuration::NNTPServerPassword() const
+{
+    return m_nntpServerPassword;
+}
+
+int Configuration::NNTPServerPort() const
+{
+    return m_nntpServerPort;
+}
+
+bool Configuration::NNTPServerSSL() const
+{
+    return m_nntpServerSSL;
+}
+
+std::string Configuration::NNTPServerUser() const
+{
+    return m_nntpServerUser;
+}
+
+void Configuration::Open(const std::string& filename)
+{
+    std::string line;
+    std::ifstream fin(filename.c_str());
+    if (!fin.is_open())
+    {
+        throw ConfigurationException(EINVAL,
+            "Could not open configuration file: " + filename
+        );
+    }
+    int line_nr = 0;
+    while(std::getline(fin,line))
+    {
+        line_nr++;
+        line = StringTrim(line);
+        // Immediately skip blank lines.
+        if (line == "") continue;
+        // Skip comments.
+        if (StringStartsWith("#",line)==true) continue;
+        // Split line in key-value pair.
+        const auto kvp = StringSplit(line, std::string{":"}, 2);
+        if (kvp.size() != 2)
+        {
+            fin.close();
+            throw ConfigurationException(EINVAL,
+                std::string("Invalid configuration in ")
+                + filename + std::string(" line ")
+                + std::to_string(line_nr)
+            );
+        }
+        const std::string key = StringToLower(kvp[0]);
+        const std::string value = StringTrim(kvp[1]);
+        if (key == "database_path")
+        {
+            m_databasePath = value;
+        }
+        else if (key == "nntp_server_host")
+        {
+            m_nntpServerHost = value;
+        }
+        else if (key == "nntp_server_pass")
+        {
+            m_nntpServerPassword = value;
+        }
+        else if (key == "nntp_server_port")
+        {
+            m_nntpServerPort = atoi(value.c_str());
+        }
+        else if (key == "nntp_server_use_ssl")
+        {
+            try
+            {
+                m_nntpServerSSL = StringToBoolean(value);
+            }
+            catch (const StringException& e)
+            {
+                fin.close();
+                throw ConfigurationException(EINVAL,
+                    std::string("Invalid configuration in ")
+                    + filename + std::string(" line ")
+                    + std::to_string(line_nr) + " - " + e.what()
+                );
+            }
+        }
+        else if (key == "nntp_server_user")
+        {
+            m_nntpServerUser = value;
+        }
+        else
+        {
+            fin.close();
+            throw ConfigurationException(EINVAL,
+                std::string("Invalid configuration in ")
+                + filename + std::string(" line ")
+                + std::to_string(line_nr)
+            );
+        }
+    }
+    fin.close();
+}
+
+} // namespace usenetsearch

src/Database.cpp

@@ -0,0 +1,72 @@
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "usenetsearch/StringUtils.h"
+#include "usenetsearch/UsenetClient.h"
+
+#include "usenetsearch/Database.h"
+
+namespace usenetsearch {
+
+Database::~Database()
+{
+    if (m_newsGroupFileInput.is_open())
+    {
+        m_newsGroupFileInput.close();
+    }
+    if (m_newsGroupFileOutput.is_open())
+    {
+        m_newsGroupFileOutput.close();
+    }
+}
+
+void Database::Open(std::filesystem::path dbPath)
+{
+    m_databasePath = dbPath;
+    if (!std::filesystem::exists(dbPath))
+    {
+        std::filesystem::create_directory(dbPath);
+    }
+    OpenNewsGroupFile();
+}
+
+void Database::OpenNewsGroupFile()
+{
+    if (m_newsGroupFileInput.is_open() && m_newsGroupFileOutput.is_open())
+    {
+        return;
+    }
+    const std::filesystem::path newsGroupFilePath =
+        m_databasePath / "newsgroups.db";
+    if (!m_newsGroupFileInput.is_open())
+    {
+        m_newsGroupFileInput.open(newsGroupFilePath, std::ios::binary);
+    }
+    if (!m_newsGroupFileOutput.is_open())
+    {
+        m_newsGroupFileOutput.open(newsGroupFilePath, std::ios::binary);
+    }
+}
+
+void Database::UpdateNewsgroupList(const std::vector<NntpListEntry>& list)
+{
+    OpenNewsGroupFile();
+    m_newsGroupFileOutput.write(
+        reinterpret_cast<const char*>(&m_databaseVersion),
+        sizeof(m_databaseVersion)
+    );
+    const size_t newsGroupCount = list.size();
+    m_newsGroupFileOutput.write(
+        reinterpret_cast<const char*>(&newsGroupCount),
+        sizeof(newsGroupCount)
+    );
+    for (const auto& entry: list)
+    {
+        m_newsGroupFileOutput << entry;
+    }
+    m_newsGroupFileOutput.flush();
+}
+
+} // namespace usenetsearch

src/StringUtils.cpp

@@ -0,0 +1,96 @@
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "usenetsearch/StringUtils.h"
+
+namespace usenetsearch {
+
+std::ostream& operator<<(std::ofstream& out, const std::string& str)
+{
+    const std::uint64_t size = str.size();
+    out.write(reinterpret_cast<const char*>(&size), sizeof(size));
+    out.write(reinterpret_cast<const char*>(str.c_str()), size);
+    return out;
+}
+
+std::ifstream& operator>>(std::ifstream& in, std::string& str)
+{
+    std::uint64_t size{0};
+    in.read(reinterpret_cast<char*>(&size), sizeof(size));
+    char buf[size];
+    in.read(buf, size);
+    buf[size] = 0;
+    str = buf;
+    return in; 
+}
+
+std::ostream& operator<<(std::ofstream& out, const std::wstring& str)
+{
+    const std::uint64_t size = str.size();
+    out.write(reinterpret_cast<const char*>(&size), sizeof(size));
+    out.write(
+        reinterpret_cast<const char*>(str.c_str()),
+        size * sizeof(wchar_t)
+    );
+    return out;
+}
+
+std::ifstream& operator>>(std::ifstream& in, std::wstring& str)
+{
+    std::uint64_t size{0};
+    in.read(reinterpret_cast<char*>(&size), sizeof(size));
+    wchar_t buf[size];
+    in.read(reinterpret_cast<char*>(buf), size * sizeof(wchar_t));
+    buf[size] = 0;
+    str = buf;
+    return in; 
+}
+
+std::string StringLeftTrim(const std::string& str)
+{
+    std::string s = str;
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+    std::not1(std::ptr_fun<int, int>(std::isspace))));
+    return s;
+}
+
+std::string StringRightTrim(const std::string& str)
+{
+    std::string s = str;
+    s.erase(std::find_if(s.rbegin(), s.rend(), 
+        std::not1(std::ptr_fun<int, int>(std::isspace))).base(), 
+        s.end());
+    return s;
+}
+
+bool StringStartsWith(const std::string& needle, const std::string& haystack)
+{
+    return (std::strncmp(haystack.c_str(),needle.c_str(),needle.size()) == 0);
+}
+
+bool StringToBoolean(const std::string& str)
+{
+    const std::string lstr = StringTrim(StringToLower(str));
+    if ((lstr == "true") || (lstr == "yes") || (lstr == "1")) return true;
+    if ((lstr == "false") || (lstr == "no") || (lstr == "0")) return false;
+    throw StringException(EINVAL,
+        "The string \"" + str + "\" is not a valid boolean value."
+    );
+}
+
+std::string StringToLower(const std::string& str)
+{
+    std::string copy = str;
+    std::transform(copy.begin(),copy.end(),copy.begin(),::tolower);
+    return copy;
+}
+
+std::string StringTrim(const std::string& str)
+{
+    return StringLeftTrim(StringRightTrim(str));
+}
+
+} // namespace usenetsearch

src/UsenetClient.cpp

@@ -1,14 +1,40 @@
 #include <codecvt>
+#include <fstream>
 #include <locale>
 #include <memory>
 #include <string>
 
 #include "usenetsearch/Except.h"
+#include "usenetsearch/StringUtils.h"
 
 #include "usenetsearch/UsenetClient.h"
 
 namespace usenetsearch {
 
+// NntpListEntry serialization -------------------------------------------------
+
+std::ostream& operator<<(std::ofstream& out, const NntpListEntry& obj)
+{
+    out.write(reinterpret_cast<const char*>(&obj.count), sizeof(obj.count));
+    out.write(reinterpret_cast<const char*>(&obj.high), sizeof(obj.high));
+    out.write(reinterpret_cast<const char*>(&obj.low), sizeof(obj.low));
+    out << obj.name;
+    out << obj.status;
+    return out;
+}
+
+std::ifstream& operator>>(std::ifstream& in, NntpListEntry& obj)
+{
+    in.read(reinterpret_cast<char*>(&obj.count), sizeof(obj.count));
+    in.read(reinterpret_cast<char*>(&obj.high), sizeof(obj.high));
+    in.read(reinterpret_cast<char*>(&obj.low), sizeof(obj.low));
+    in >> obj.name;
+    in >> obj.status;
+    return in;
+}
+
+// UsenetClient class ----------------------------------------------------------
+
 void UsenetClient::Authenticate(
     const std::wstring& user,
     const std::wstring& password)
@@ -80,6 +106,42 @@ bool UsenetClient::IsError(const NntpMessage& msg) const
     return false;
 }
 
+std::vector<NntpListEntry> UsenetClient::List()
+{
+    Write(L"LIST COUNTS\r\n");
+    /* In response, we should get a 215 response followed by the list of news
+       groups ending in a period on it's own line. */
+    const auto response = ReadLine();
+    if (IsError(response))
+    {
+        throw UsenetClientException(
+            response.code,
+            "Failed to fetch newsgroup list from server, "
+            + std::string{"server responded with: "} 
+            + m_conv.to_bytes(response.message)
+        );
+    }
+    const auto listStr = ReadUntil(L"\r\n.\r\n");
+    // parse the list.
+    const auto lines = StringSplit(listStr, std::wstring{L"\r\n"});
+    std::vector<NntpListEntry> result;
+    for (const auto& line: lines)
+    {
+        NntpListEntry entry;
+        const auto fields = StringSplit(line, std::wstring{L" "});
+        if (fields.size() == 5)
+        {
+            entry.name = fields[0];
+            entry.high = std::stoul(fields[1]);
+            entry.low = std::stoul(fields[2]);
+            entry.count = std::stoul(fields[3]);
+            entry.status = fields[4];
+            result.emplace_back(entry);
+        }
+    }
+    return result;
+}
+
 NntpMessage UsenetClient::ReadLine()
 {
     NntpMessage result{};

src/main.cpp

@@ -1,30 +1,85 @@
 #include <iostream>
 #include <memory>
 
+#include "usenetsearch/Configuration.h"
+#include "usenetsearch/Database.h"
+#include "usenetsearch/Except.h"
+#include "usenetsearch/StringUtils.h"
 #include "usenetsearch/UsenetClient.h"
 
+using usenetsearch::StringStartsWith;
+
+void Usage(const std::string& programName)
+{
+    std::cout << programName;
+    std::cout << "\t";
+    std::cout << "[-c <config filename>] ";
+    std::cout << "[-h] " << std::endl << std::endl;
+    std::cout << "-c <file>\tSets configuration file to use" << std::endl;
+    std::cout << "-h\tShow help (this text)." << std::endl;
+    std::cout << std::endl;
+}
+
 int main(int argc, char* argv[])
 {
 
-    (void) argc;
-    (void) argv;
+    std::string configFile{"config.json"};
 
-    std::string host = "news.newshosting.com";
-    std::uint16_t port = 443;
-    bool useSSL = true;
+    // Parse args.
+    for (int argn = 1; argn != argc; ++argn)
+    {
+        std::string curr_opt = argv[argn];
+        std::string next_opt = "";
+        if (argn+1 < argc) next_opt=argv[argn+1];
+        if (curr_opt == "-c")
+        {
+            if ((next_opt == "") or (StringStartsWith("-", next_opt)))
+            {
+                std::cerr << "Missing argument to -c option." << std::endl;
+                Usage(argv[0]);
+                return 1;
+            }
+            argn++;
+            configFile = argv[argn];
+        }
+        else if (curr_opt == "-h")
+        {
+            Usage(argv[0]);
+            return 0;
+        }
+    }
 
+    // Read config, setup db
+    usenetsearch::Configuration config;
+    config.Open(configFile);
+    usenetsearch::Database db;
+    db.Open(config.DatabasePath());
+
+    // Start nntp client.
     usenetsearch::UsenetClient client;
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv;
     try 
     {
-        client.Connect(host, port, useSSL);
-        client.Authenticate(L"xxxxxxx", L"yyyyy");
+        client.Connect(
+            config.NNTPServerHost(),
+            config.NNTPServerPort(),
+            config.NNTPServerSSL()
+        );
+        client.Authenticate(
+            conv.from_bytes(config.NNTPServerUser()),
+            conv.from_bytes(config.NNTPServerPassword())
+        );
+
+        // Just testing the list command for now.
+        const auto list = client.List();
+        db.UpdateNewsgroupList(list);
+        std::cout << "Number of newsgroups in newsgroup list: "
+                  << list.size() << std::endl;
     }
-    catch (const std::exception& e)
+    catch (const usenetsearch::UsenetSearchException& e)
     {
         std::cerr << e.what() << std::endl;;
         return 1;
     }
-    std::cout << "success." << std::endl;
-
     return 0;
 }

usenetsearch.example.conf

@@ -0,0 +1,9 @@
+# NNTP server configuration details
+nntp_server_host: news.example.com
+nntp_server_port: 119
+nntp_server_user: configureMe
+nntp_server_pass: configureMe
+nntp_server_use_ssl: no
+
+# Index database configuration details
+database_path: ./db