From a68c46299c8e1e92bd9bf0b5c8f4478500d62fc7 Mon Sep 17 00:00:00 2001 From: John Sennesael Date: Mon, 20 Sep 2021 19:48:49 -0500 Subject: [PATCH] Configuration file, arg parsing, database serialization,... --- .gitignore | 1 + CMakeLists.txt | 3 + include/usenetsearch/Configuration.h | 40 +++++++++ include/usenetsearch/Database.h | 31 +++++++ include/usenetsearch/SSLConnection.h | 6 +- include/usenetsearch/StringUtils.h | 58 +++++++++++++ include/usenetsearch/TcpConnection.h | 4 +- include/usenetsearch/UsenetClient.h | 14 +++ src/Configuration.cpp | 122 +++++++++++++++++++++++++++ src/Database.cpp | 72 ++++++++++++++++ src/StringUtils.cpp | 96 +++++++++++++++++++++ src/UsenetClient.cpp | 62 ++++++++++++++ src/main.cpp | 75 +++++++++++++--- usenetsearch.example.conf | 9 ++ 14 files changed, 577 insertions(+), 16 deletions(-) create mode 100644 include/usenetsearch/Configuration.h create mode 100644 include/usenetsearch/Database.h create mode 100644 include/usenetsearch/StringUtils.h create mode 100644 src/Configuration.cpp create mode 100644 src/Database.cpp create mode 100644 src/StringUtils.cpp create mode 100644 usenetsearch.example.conf diff --git a/.gitignore b/.gitignore index cbb78c1..16d09d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.*~ *.sw* build/* +usenetsearch.conf diff --git a/CMakeLists.txt b/CMakeLists.txt index 37f208a..6a737d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,11 +18,14 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(OpenSSL REQUIRED) add_executable(usenetsearch + "src/Configuration.cpp" + "src/Database.cpp" "src/Dns.cpp" "src/Except.cpp" "src/IoSocket.cpp" "src/main.cpp" "src/SSLConnection.cpp" + "src/StringUtils.cpp" "src/TcpConnection.cpp" "src/UsenetClient.cpp" ) diff --git a/include/usenetsearch/Configuration.h b/include/usenetsearch/Configuration.h new file mode 100644 index 0000000..65c2405 --- /dev/null +++ b/include/usenetsearch/Configuration.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include "usenetsearch/Except.h" + +namespace usenetsearch { + +struct ConfigurationException: public UsenetSearchException +{ + ConfigurationException(int errorCode, const std::string& message): + UsenetSearchException(errorCode, message){} + + virtual ~ConfigurationException() = default; +}; + +class Configuration +{ + + std::string m_nntpServerHost{"127.0.0.1"}; + std::string m_nntpServerPassword{"password"}; + int m_nntpServerPort{119}; + bool m_nntpServerSSL{false}; + std::string m_nntpServerUser{"username"}; + std::filesystem::path m_databasePath{"./db"}; + +public: + + std::filesystem::path DatabasePath() const; + std::string NNTPServerHost() const; + std::string NNTPServerPassword() const; + int NNTPServerPort() const; + bool NNTPServerSSL() const; + std::string NNTPServerUser() const; + void Open(const std::string& filename); + +}; + +} // namespace usenetsearch diff --git a/include/usenetsearch/Database.h b/include/usenetsearch/Database.h new file mode 100644 index 0000000..984ac73 --- /dev/null +++ b/include/usenetsearch/Database.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +#include "usenetsearch/UsenetClient.h" + +namespace usenetsearch { + +static constexpr const std::uint64_t DatabaseVersion{1}; + +class Database{ + + std::filesystem::path m_databasePath; + std::uint64_t m_databaseVersion{0}; + std::ifstream m_newsGroupFileInput; + std::ofstream m_newsGroupFileOutput; + + void OpenNewsGroupFile(); + +public: + + ~Database(); + void Open(std::filesystem::path dbPath); + void UpdateNewsgroupList(const std::vector& list); + +}; + +} // namespace usenetsearch diff --git a/include/usenetsearch/SSLConnection.h b/include/usenetsearch/SSLConnection.h index 582afc9..af2f4d7 100644 --- a/include/usenetsearch/SSLConnection.h +++ b/include/usenetsearch/SSLConnection.h @@ -21,8 +21,6 @@ class SSLConnection : public IoSocket { enum class SSLReturnState{ RETRY, SUCCESS }; - std::chrono::milliseconds m_connectionTimeout{10000}; - std::chrono::milliseconds m_ioTimeout{10000}; std::shared_ptr m_ssl; std::shared_ptr m_sslContext; std::unique_ptr m_tcpConnection; @@ -34,8 +32,8 @@ public: SSLConnection(std::unique_ptr connection); void Connect(); void Disconnect(); - std::string Read(size_t amount); - void Write(const std::string& data); + virtual std::string Read(size_t amount) override; + virtual void Write(const std::string& data) override; }; } // namespace usenetsearch diff --git a/include/usenetsearch/StringUtils.h b/include/usenetsearch/StringUtils.h new file mode 100644 index 0000000..d7247a7 --- /dev/null +++ b/include/usenetsearch/StringUtils.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include + +#include "usenetsearch/Except.h" + +namespace usenetsearch { + +std::ostream& operator<<(std::ofstream& out, const std::string& str); +std::ifstream& operator>>(std::ifstream& in, std::string& str); +std::ostream& operator<<(std::ofstream& out, const std::wstring& str); +std::ifstream& operator>>(std::ifstream& in, std::wstring& str); + +struct StringException: public UsenetSearchException +{ + StringException(int errorCode, const std::string& message): + UsenetSearchException(errorCode, message){} + + virtual ~StringException() = default; +}; + +template +std::vector StringSplit( + const T& str, + const T& delim, + int max_elem = -1 +) +{ + size_t pos=0; + T s = str; + std::vector result; + int tokens = 1; + while (((pos = s.find(delim)) != T::npos) + and ((max_elem < 0) or (tokens != max_elem))) + { + result.push_back(s.substr(0,pos)); + s.erase(0, pos + delim.length()); + tokens++; + } + if (max_elem != 0) result.push_back(s); + return result; +} + +std::string StringLeftTrim(const std::string& str); + +std::string StringRightTrim(const std::string& str); + +bool StringStartsWith(const std::string& needle, const std::string& haystack); + +std::string StringTrim(const std::string& str); + +std::string StringToLower(const std::string& str); + +bool StringToBoolean(const std::string& str); + +} // namespace usenetsearch diff --git a/include/usenetsearch/TcpConnection.h b/include/usenetsearch/TcpConnection.h index 1025664..1acdb97 100644 --- a/include/usenetsearch/TcpConnection.h +++ b/include/usenetsearch/TcpConnection.h @@ -28,8 +28,8 @@ public: void Connect(const std::string& host, std::uint16_t port); void Disconnect(); int FileDescriptor() const; - std::string Read(size_t amount); - void Write(const std::string& data); + virtual std::string Read(size_t amount) override; + virtual void Write(const std::string& data) override; }; diff --git a/include/usenetsearch/UsenetClient.h b/include/usenetsearch/UsenetClient.h index 0c00671..22e756e 100644 --- a/include/usenetsearch/UsenetClient.h +++ b/include/usenetsearch/UsenetClient.h @@ -2,9 +2,11 @@ #include #include +#include #include #include #include +#include #include "usenetsearch/SSLConnection.h" #include "usenetsearch/TcpConnection.h" @@ -25,6 +27,17 @@ struct NntpMessage std::wstring message; }; +struct NntpListEntry +{ + std::wstring name; + std::uint64_t high; + std::uint64_t low; + std::uint64_t count; + std::wstring status; +}; +std::ostream& operator<<(std::ofstream& out, const NntpListEntry& obj); +std::ifstream& operator>>(std::ifstream& in, NntpListEntry& obj); + class UsenetClient { std::wstring_convert> m_conv; @@ -45,6 +58,7 @@ public: std::uint16_t port, bool useSSL = false ); + std::vector List(); }; diff --git a/src/Configuration.cpp b/src/Configuration.cpp new file mode 100644 index 0000000..9172fbd --- /dev/null +++ b/src/Configuration.cpp @@ -0,0 +1,122 @@ +#include +#include +#include + +#include "usenetsearch/StringUtils.h" + +#include "usenetsearch/Configuration.h" + +namespace usenetsearch { + +std::filesystem::path Configuration::DatabasePath() const +{ + return m_databasePath; +} + +std::string Configuration::NNTPServerHost() const +{ + return m_nntpServerHost; +} + +std::string Configuration::NNTPServerPassword() const +{ + return m_nntpServerPassword; +} + +int Configuration::NNTPServerPort() const +{ + return m_nntpServerPort; +} + +bool Configuration::NNTPServerSSL() const +{ + return m_nntpServerSSL; +} + +std::string Configuration::NNTPServerUser() const +{ + return m_nntpServerUser; +} + +void Configuration::Open(const std::string& filename) +{ + std::string line; + std::ifstream fin(filename.c_str()); + if (!fin.is_open()) + { + throw ConfigurationException(EINVAL, + "Could not open configuration file: " + filename + ); + } + int line_nr = 0; + while(std::getline(fin,line)) + { + line_nr++; + line = StringTrim(line); + // Immediately skip blank lines. + if (line == "") continue; + // Skip comments. + if (StringStartsWith("#",line)==true) continue; + // Split line in key-value pair. + const auto kvp = StringSplit(line, std::string{":"}, 2); + if (kvp.size() != 2) + { + fin.close(); + throw ConfigurationException(EINVAL, + std::string("Invalid configuration in ") + + filename + std::string(" line ") + + std::to_string(line_nr) + ); + } + const std::string key = StringToLower(kvp[0]); + const std::string value = StringTrim(kvp[1]); + if (key == "database_path") + { + m_databasePath = value; + } + else if (key == "nntp_server_host") + { + m_nntpServerHost = value; + } + else if (key == "nntp_server_pass") + { + m_nntpServerPassword = value; + } + else if (key == "nntp_server_port") + { + m_nntpServerPort = atoi(value.c_str()); + } + else if (key == "nntp_server_use_ssl") + { + try + { + m_nntpServerSSL = StringToBoolean(value); + } + catch (const StringException& e) + { + fin.close(); + throw ConfigurationException(EINVAL, + std::string("Invalid configuration in ") + + filename + std::string(" line ") + + std::to_string(line_nr) + " - " + e.what() + ); + } + } + else if (key == "nntp_server_user") + { + m_nntpServerUser = value; + } + else + { + fin.close(); + throw ConfigurationException(EINVAL, + std::string("Invalid configuration in ") + + filename + std::string(" line ") + + std::to_string(line_nr) + ); + } + } + fin.close(); +} + +} // namespace usenetsearch diff --git a/src/Database.cpp b/src/Database.cpp new file mode 100644 index 0000000..0bec1cb --- /dev/null +++ b/src/Database.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + +#include "usenetsearch/StringUtils.h" +#include "usenetsearch/UsenetClient.h" + +#include "usenetsearch/Database.h" + +namespace usenetsearch { + +Database::~Database() +{ + if (m_newsGroupFileInput.is_open()) + { + m_newsGroupFileInput.close(); + } + if (m_newsGroupFileOutput.is_open()) + { + m_newsGroupFileOutput.close(); + } +} + +void Database::Open(std::filesystem::path dbPath) +{ + m_databasePath = dbPath; + if (!std::filesystem::exists(dbPath)) + { + std::filesystem::create_directory(dbPath); + } + OpenNewsGroupFile(); +} + +void Database::OpenNewsGroupFile() +{ + if (m_newsGroupFileInput.is_open() && m_newsGroupFileOutput.is_open()) + { + return; + } + const std::filesystem::path newsGroupFilePath = + m_databasePath / "newsgroups.db"; + if (!m_newsGroupFileInput.is_open()) + { + m_newsGroupFileInput.open(newsGroupFilePath, std::ios::binary); + } + if (!m_newsGroupFileOutput.is_open()) + { + m_newsGroupFileOutput.open(newsGroupFilePath, std::ios::binary); + } +} + +void Database::UpdateNewsgroupList(const std::vector& list) +{ + OpenNewsGroupFile(); + m_newsGroupFileOutput.write( + reinterpret_cast(&m_databaseVersion), + sizeof(m_databaseVersion) + ); + const size_t newsGroupCount = list.size(); + m_newsGroupFileOutput.write( + reinterpret_cast(&newsGroupCount), + sizeof(newsGroupCount) + ); + for (const auto& entry: list) + { + m_newsGroupFileOutput << entry; + } + m_newsGroupFileOutput.flush(); +} + +} // namespace usenetsearch diff --git a/src/StringUtils.cpp b/src/StringUtils.cpp new file mode 100644 index 0000000..0feeca0 --- /dev/null +++ b/src/StringUtils.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include + +#include "usenetsearch/StringUtils.h" + +namespace usenetsearch { + +std::ostream& operator<<(std::ofstream& out, const std::string& str) +{ + const std::uint64_t size = str.size(); + out.write(reinterpret_cast(&size), sizeof(size)); + out.write(reinterpret_cast(str.c_str()), size); + return out; +} + +std::ifstream& operator>>(std::ifstream& in, std::string& str) +{ + std::uint64_t size{0}; + in.read(reinterpret_cast(&size), sizeof(size)); + char buf[size]; + in.read(buf, size); + buf[size] = 0; + str = buf; + return in; +} + +std::ostream& operator<<(std::ofstream& out, const std::wstring& str) +{ + const std::uint64_t size = str.size(); + out.write(reinterpret_cast(&size), sizeof(size)); + out.write( + reinterpret_cast(str.c_str()), + size * sizeof(wchar_t) + ); + return out; +} + +std::ifstream& operator>>(std::ifstream& in, std::wstring& str) +{ + std::uint64_t size{0}; + in.read(reinterpret_cast(&size), sizeof(size)); + wchar_t buf[size]; + in.read(reinterpret_cast(buf), size * sizeof(wchar_t)); + buf[size] = 0; + str = buf; + return in; +} + +std::string StringLeftTrim(const std::string& str) +{ + std::string s = str; + s.erase(s.begin(), std::find_if(s.begin(), s.end(), + std::not1(std::ptr_fun(std::isspace)))); + return s; +} + +std::string StringRightTrim(const std::string& str) +{ + std::string s = str; + s.erase(std::find_if(s.rbegin(), s.rend(), + std::not1(std::ptr_fun(std::isspace))).base(), + s.end()); + return s; +} + +bool StringStartsWith(const std::string& needle, const std::string& haystack) +{ + return (std::strncmp(haystack.c_str(),needle.c_str(),needle.size()) == 0); +} + +bool StringToBoolean(const std::string& str) +{ + const std::string lstr = StringTrim(StringToLower(str)); + if ((lstr == "true") || (lstr == "yes") || (lstr == "1")) return true; + if ((lstr == "false") || (lstr == "no") || (lstr == "0")) return false; + throw StringException(EINVAL, + "The string \"" + str + "\" is not a valid boolean value." + ); +} + +std::string StringToLower(const std::string& str) +{ + std::string copy = str; + std::transform(copy.begin(),copy.end(),copy.begin(),::tolower); + return copy; +} + +std::string StringTrim(const std::string& str) +{ + return StringLeftTrim(StringRightTrim(str)); +} + +} // namespace usenetsearch diff --git a/src/UsenetClient.cpp b/src/UsenetClient.cpp index db59940..f312c7a 100644 --- a/src/UsenetClient.cpp +++ b/src/UsenetClient.cpp @@ -1,14 +1,40 @@ #include +#include #include #include #include #include "usenetsearch/Except.h" +#include "usenetsearch/StringUtils.h" #include "usenetsearch/UsenetClient.h" namespace usenetsearch { +// NntpListEntry serialization ------------------------------------------------- + +std::ostream& operator<<(std::ofstream& out, const NntpListEntry& obj) +{ + out.write(reinterpret_cast(&obj.count), sizeof(obj.count)); + out.write(reinterpret_cast(&obj.high), sizeof(obj.high)); + out.write(reinterpret_cast(&obj.low), sizeof(obj.low)); + out << obj.name; + out << obj.status; + return out; +} + +std::ifstream& operator>>(std::ifstream& in, NntpListEntry& obj) +{ + in.read(reinterpret_cast(&obj.count), sizeof(obj.count)); + in.read(reinterpret_cast(&obj.high), sizeof(obj.high)); + in.read(reinterpret_cast(&obj.low), sizeof(obj.low)); + in >> obj.name; + in >> obj.status; + return in; +} + +// UsenetClient class ---------------------------------------------------------- + void UsenetClient::Authenticate( const std::wstring& user, const std::wstring& password) @@ -80,6 +106,42 @@ bool UsenetClient::IsError(const NntpMessage& msg) const return false; } +std::vector UsenetClient::List() +{ + Write(L"LIST COUNTS\r\n"); + /* In response, we should get a 215 response followed by the list of news + groups ending in a period on it's own line. */ + const auto response = ReadLine(); + if (IsError(response)) + { + throw UsenetClientException( + response.code, + "Failed to fetch newsgroup list from server, " + + std::string{"server responded with: "} + + m_conv.to_bytes(response.message) + ); + } + const auto listStr = ReadUntil(L"\r\n.\r\n"); + // parse the list. + const auto lines = StringSplit(listStr, std::wstring{L"\r\n"}); + std::vector result; + for (const auto& line: lines) + { + NntpListEntry entry; + const auto fields = StringSplit(line, std::wstring{L" "}); + if (fields.size() == 5) + { + entry.name = fields[0]; + entry.high = std::stoul(fields[1]); + entry.low = std::stoul(fields[2]); + entry.count = std::stoul(fields[3]); + entry.status = fields[4]; + result.emplace_back(entry); + } + } + return result; +} + NntpMessage UsenetClient::ReadLine() { NntpMessage result{}; diff --git a/src/main.cpp b/src/main.cpp index b336ddf..0ab639d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,30 +1,85 @@ #include #include +#include "usenetsearch/Configuration.h" +#include "usenetsearch/Database.h" +#include "usenetsearch/Except.h" +#include "usenetsearch/StringUtils.h" #include "usenetsearch/UsenetClient.h" +using usenetsearch::StringStartsWith; + +void Usage(const std::string& programName) +{ + std::cout << programName; + std::cout << "\t"; + std::cout << "[-c ] "; + std::cout << "[-h] " << std::endl << std::endl; + std::cout << "-c \tSets configuration file to use" << std::endl; + std::cout << "-h\tShow help (this text)." << std::endl; + std::cout << std::endl; +} + int main(int argc, char* argv[]) { - (void) argc; - (void) argv; + std::string configFile{"config.json"}; - std::string host = "news.newshosting.com"; - std::uint16_t port = 443; - bool useSSL = true; + // Parse args. + for (int argn = 1; argn != argc; ++argn) + { + std::string curr_opt = argv[argn]; + std::string next_opt = ""; + if (argn+1 < argc) next_opt=argv[argn+1]; + if (curr_opt == "-c") + { + if ((next_opt == "") or (StringStartsWith("-", next_opt))) + { + std::cerr << "Missing argument to -c option." << std::endl; + Usage(argv[0]); + return 1; + } + argn++; + configFile = argv[argn]; + } + else if (curr_opt == "-h") + { + Usage(argv[0]); + return 0; + } + } + // Read config, setup db + usenetsearch::Configuration config; + config.Open(configFile); + usenetsearch::Database db; + db.Open(config.DatabasePath()); + + // Start nntp client. usenetsearch::UsenetClient client; + std::wstring_convert> conv; try { - client.Connect(host, port, useSSL); - client.Authenticate(L"xxxxxxx", L"yyyyy"); + client.Connect( + config.NNTPServerHost(), + config.NNTPServerPort(), + config.NNTPServerSSL() + ); + client.Authenticate( + conv.from_bytes(config.NNTPServerUser()), + conv.from_bytes(config.NNTPServerPassword()) + ); + + // Just testing the list command for now. + const auto list = client.List(); + db.UpdateNewsgroupList(list); + std::cout << "Number of newsgroups in newsgroup list: " + << list.size() << std::endl; } - catch (const std::exception& e) + catch (const usenetsearch::UsenetSearchException& e) { std::cerr << e.what() << std::endl;; return 1; } - std::cout << "success." << std::endl; - return 0; } diff --git a/usenetsearch.example.conf b/usenetsearch.example.conf new file mode 100644 index 0000000..cd4da7b --- /dev/null +++ b/usenetsearch.example.conf @@ -0,0 +1,9 @@ +# NNTP server configuration details +nntp_server_host: news.example.com +nntp_server_port: 119 +nntp_server_user: configureMe +nntp_server_pass: configureMe +nntp_server_use_ssl: no + +# Index database configuration details +database_path: ./db