diff --git a/include/usenetsearch/Application.h b/include/usenetsearch/Application.h index e693f18..d391407 100644 --- a/include/usenetsearch/Application.h +++ b/include/usenetsearch/Application.h @@ -56,6 +56,8 @@ class Application std::string m_configFile{"usenetsearch.conf"}; Database m_db; Filter m_filter; + /// Holds a pointer to the singleton instance. + static std::unique_ptr m_instance; void ExecuteCustomOption( std::shared_ptr&, @@ -93,10 +95,15 @@ public: std::string defaultValue = "" ); bool CanRun() const; + /** + * Gets an instance to the application singleton. + */ + static Application& Get(); Configuration& GetConfig(); Database& GetDb(); Filter& GetFilter(); bool Init(int argc, char* argv[]); + bool ShouldStop() const; void Usage(const std::string& programName); }; diff --git a/include/usenetsearch/Configuration.h b/include/usenetsearch/Configuration.h index a533d4a..1002ebb 100644 --- a/include/usenetsearch/Configuration.h +++ b/include/usenetsearch/Configuration.h @@ -27,8 +27,8 @@ namespace usenetsearch { struct ConfigurationException: public UsenetSearchException { - ConfigurationException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + ConfigurationException(const std::string& message): + UsenetSearchException(message){} virtual ~ConfigurationException() = default; }; diff --git a/include/usenetsearch/Database.h b/include/usenetsearch/Database.h index c2cb919..6a8c6b7 100644 --- a/include/usenetsearch/Database.h +++ b/include/usenetsearch/Database.h @@ -47,15 +47,14 @@ std::fstream& operator>>(std::fstream& in, ArticleEntry& obj); struct DatabaseException: public UsenetSearchException { - DatabaseException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + DatabaseException(const std::string& message): + UsenetSearchException(message){} virtual ~DatabaseException() = default; }; class Database { - Application& m_app; std::filesystem::path m_databasePath; std::uint64_t m_databaseVersion{DatabaseVersion}; std::vector m_lockedFiles; @@ -91,8 +90,9 @@ class Database public: - explicit Database(Application& app); - ~Database(); + Database() = default; + ~Database() = default; + std::unique_ptr FindNntpEntry(std::uint64_t id); std::unique_ptr FindNntpEntry(const std::string& subject); std::uint32_t GetLastIndexedArticle(std::uint64_t newsgroupID); std::unique_ptr> LoadNewsgroupList(); diff --git a/include/usenetsearch/Dns.h b/include/usenetsearch/Dns.h index e6b9bd8..22de23e 100644 --- a/include/usenetsearch/Dns.h +++ b/include/usenetsearch/Dns.h @@ -29,8 +29,8 @@ namespace usenetsearch { struct DnsResolveException: public UsenetSearchException { - DnsResolveException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + DnsResolveException(const std::string& message): + UsenetSearchException(message){} virtual ~DnsResolveException() = default; }; diff --git a/include/usenetsearch/Except.h b/include/usenetsearch/Except.h index b5a97cc..92dac52 100644 --- a/include/usenetsearch/Except.h +++ b/include/usenetsearch/Except.h @@ -23,15 +23,12 @@ namespace usenetsearch { class UsenetSearchException: public std::exception { - int m_errorCode; std::string m_message; public: - UsenetSearchException(int errorCode, const std::string& message); + UsenetSearchException(const std::string& message); + virtual ~UsenetSearchException() = default; - - int Code() const; - virtual const char* what() const noexcept override; }; diff --git a/include/usenetsearch/Indexer.h b/include/usenetsearch/Indexer.h index 94bf682..78f532e 100644 --- a/include/usenetsearch/Indexer.h +++ b/include/usenetsearch/Indexer.h @@ -17,7 +17,7 @@ #pragma once -#include "usenetsearch/Application.h" +#include "usenetsearch/Database.h" #include "usenetsearch/Filter.h" #include "usenetsearch/UsenetClient.h" #include "usenetsearch/ThreadPool.h" @@ -61,14 +61,13 @@ typedef std::vector SearchResults; class Indexer { - Application& m_app; UsenetClient& m_client; std::wstring_convert> m_conv; ThreadPool m_threads; public: - Indexer(Application& app, UsenetClient& client); + Indexer(UsenetClient& client); void Connect(); void Index(const std::vector& newsgroups); diff --git a/include/usenetsearch/Logger.h b/include/usenetsearch/Logger.h index 748172c..ca5a4e8 100644 --- a/include/usenetsearch/Logger.h +++ b/include/usenetsearch/Logger.h @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace usenetsearch { @@ -32,6 +33,8 @@ class Logger /// Holds a pointer to the singleton instance. static std::unique_ptr m_instance; + std::mutex m_stdOutLock; + std::mutex m_stdErrLock; public: /** @@ -47,28 +50,28 @@ public: /** * Gets an instance to the logger. */ - static Logger& get(); + static Logger& Get(); /** * Logs a debug message. * * @param[in] msg Message to be logged. */ - void debug(const std::string& msg) const; + void Debug(const std::string& func, const std::string& msg); /** * Logs an info message. * * @param[in] msg Message to be logged. */ - void info(const std::string& msg) const; + void Info(const std::string& func, const std::string& msg); /** * Logs an error message. * * @param[in] msg Message to be logged. */ - void error(const std::string& msg) const; + void Error(const std::string& func, const std::string& msg); /** * Logs an error message and throws. @@ -77,14 +80,18 @@ public: * @param[in] msg Error message to be logged. */ template - inline void fatal(const std::string& msg) const + inline void Fatal(const std::string& func, const std::string& msg) { static_assert( std::is_base_of::value, "EXCEPTION must be an std::exception or derrived child."); - error(msg); + Error(func, msg); throw EXCEPTION(msg); }; }; +std::string LogName(const std::string& module, const std::string& func); + +#define LOGID(module) LogName(module, __FUNCTION__) + } // namespace usenetsearch diff --git a/include/usenetsearch/SSLConnection.h b/include/usenetsearch/SSLConnection.h index 4a3f8c0..acab3d3 100644 --- a/include/usenetsearch/SSLConnection.h +++ b/include/usenetsearch/SSLConnection.h @@ -28,8 +28,8 @@ namespace usenetsearch { struct SSLException: public UsenetSearchException { - SSLException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + SSLException(const std::string& message): + UsenetSearchException(message){} virtual ~SSLException() = default; }; diff --git a/include/usenetsearch/Serialize.h b/include/usenetsearch/Serialize.h index 2b631e9..fe80829 100644 --- a/include/usenetsearch/Serialize.h +++ b/include/usenetsearch/Serialize.h @@ -33,16 +33,16 @@ struct NntpListEntry; struct FileIOException: public UsenetSearchException { - FileIOException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + FileIOException(const std::string& message): + UsenetSearchException(message){} virtual ~FileIOException() = default; }; struct SerializeException: public UsenetSearchException { - SerializeException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + SerializeException(const std::string& message): + UsenetSearchException(message){} virtual ~SerializeException() = default; }; @@ -125,5 +125,4 @@ SerializableFile& operator>>(SerializableFile& in, NntpHeader& obj); SerializableFile& operator<<(SerializableFile& out, const NntpListEntry& obj); SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj); - } // namespace usenetsearch diff --git a/include/usenetsearch/StringUtils.h b/include/usenetsearch/StringUtils.h index 6f4470b..b7d68d7 100644 --- a/include/usenetsearch/StringUtils.h +++ b/include/usenetsearch/StringUtils.h @@ -29,8 +29,8 @@ namespace usenetsearch { struct StringException: public UsenetSearchException { - StringException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + StringException(const std::string& message): + UsenetSearchException(message){} virtual ~StringException() = default; }; diff --git a/include/usenetsearch/TcpConnection.h b/include/usenetsearch/TcpConnection.h index 3391f96..e02b8aa 100644 --- a/include/usenetsearch/TcpConnection.h +++ b/include/usenetsearch/TcpConnection.h @@ -27,8 +27,8 @@ namespace usenetsearch { struct SocketException: public UsenetSearchException { - SocketException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + SocketException(const std::string& message): + UsenetSearchException(message){} virtual ~SocketException() = default; }; diff --git a/include/usenetsearch/UsenetClient.h b/include/usenetsearch/UsenetClient.h index b7239ef..0dd6b22 100644 --- a/include/usenetsearch/UsenetClient.h +++ b/include/usenetsearch/UsenetClient.h @@ -36,8 +36,8 @@ class Application; struct UsenetClientException: public UsenetSearchException { - UsenetClientException(int errorCode, const std::string& message): - UsenetSearchException(errorCode, message){} + UsenetClientException(const std::string& message): + UsenetSearchException(message){} virtual ~UsenetClientException() = default; }; @@ -77,7 +77,6 @@ struct NntpListEntry class UsenetClient { - Application& m_app; std::unique_ptr m_ssl; std::unique_ptr m_tcp; bool m_useSSL{false}; @@ -89,7 +88,7 @@ class UsenetClient public: - UsenetClient(Application& app); + UsenetClient() = default; void Authenticate(const std::wstring& user, const std::wstring& password); diff --git a/src/Application.cpp b/src/Application.cpp index f1feab0..67c32e5 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -17,8 +17,10 @@ #include "usenetsearch/Application.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/StringUtils.h" +#include #include #include #include @@ -26,6 +28,15 @@ namespace usenetsearch { +sig_atomic_t _signalled_to_stop{0}; + +void sigHandler(int) +{ + _signalled_to_stop = 1; +} + +std::unique_ptr Application::m_instance; + void Application::AddBooleanOption( char option, const std::string& help, @@ -88,6 +99,25 @@ void Application::AddStringOption( m_commandLineArguments.emplace_back(std::move(val)); } +Application& Application::Get() +{ + if (m_instance == nullptr) + { + m_instance = std::make_unique(); + } + return *m_instance; +} + +bool Application::ShouldStop() const +{ + const bool result = _signalled_to_stop == 1; + if (result) + { + Logger::Get().Info(LOGID("Application"), "Been signalled to stop."); + } + return result; +} + void Application::Usage(const std::string& programName) { std::cout << "UsenetSearch - usenet search indexer" << std::endl; @@ -147,7 +177,7 @@ void Application::Usage(const std::string& programName) std::cout << std::endl; } -Application::Application() : m_db(*this), m_filter(m_config) +Application::Application() : m_filter(m_config) { std::cout.setf(std::ios::unitbuf); } @@ -249,6 +279,11 @@ bool Application::Init(int argc, char* argv[]) { ParseArgs(argc, argv); if (!m_canRun) return false; + // Install signal handlers. + void (*prev_handler)(int); + prev_handler = signal(SIGINT, sigHandler); + prev_handler = signal(SIGTERM, sigHandler); + prev_handler = signal(SIGTSTP, sigHandler); // Read config, setup db m_config.Open(m_configFile); m_db.MaxTreeDepth(m_config.MaxTreeDepth()); diff --git a/src/Configuration.cpp b/src/Configuration.cpp index 1560f82..53c8ea2 100644 --- a/src/Configuration.cpp +++ b/src/Configuration.cpp @@ -17,6 +17,7 @@ #include "usenetsearch/Configuration.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/ScopeExit.h" #include "usenetsearch/StringUtils.h" @@ -103,7 +104,8 @@ void Configuration::Open(const std::string& filename) std::ifstream fin(filename.c_str()); if (!fin.is_open()) { - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), "Could not open configuration file: " + filename ); } @@ -121,7 +123,8 @@ void Configuration::Open(const std::string& filename) const auto kvp = StringSplit(line, std::string{":"}, 2); if (kvp.size() != 2) { - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), std::string("Invalid configuration in ") + filename + std::string(" line ") + std::to_string(line_nr) @@ -158,7 +161,8 @@ void Configuration::Open(const std::string& filename) } catch (const std::regex_error& e) { - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), std::string("Invalid configuration in ") + filename + std::string(" line ") + std::to_string(line_nr) + " : Regular expression \"" @@ -175,7 +179,8 @@ void Configuration::Open(const std::string& filename) } catch (const std::regex_error& e) { - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), std::string("Invalid configuration in ") + filename + std::string(" line ") + std::to_string(line_nr) + " : Regular expression \"" @@ -228,7 +233,8 @@ void Configuration::Open(const std::string& filename) catch (const StringException& e) { fin.close(); - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), std::string("Invalid configuration in ") + filename + std::string(" line ") + std::to_string(line_nr) + " - " + e.what() @@ -241,7 +247,8 @@ void Configuration::Open(const std::string& filename) } else { - throw ConfigurationException(EINVAL, + Logger::Get().Fatal( + LOGID("Configuration"), std::string("Invalid configuration in ") + filename + std::string(" line ") + std::to_string(line_nr) diff --git a/src/Database.cpp b/src/Database.cpp index 76f6edb..e6fb090 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -18,13 +18,13 @@ #include "usenetsearch/Database.h" #include "usenetsearch/Application.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/StringUtils.h" -#include "usenetsearch/UsenetClient.h" #include "usenetsearch/ScopeExit.h" #include "usenetsearch/Serialize.h" +#include "usenetsearch/UsenetClient.h" #include -#include #include #include #include @@ -36,27 +36,42 @@ namespace usenetsearch { // Database class -------------------------------------------------------------- -Database::Database(Application& app): m_app(app) -{ -} - -Database::~Database() -{ -} - void Database::CheckDbVersion(const SerializableFile& f) const { f.Seek(0); const std::uint64_t ver = f.ReadInt64(); if (ver != m_databaseVersion) { - throw DatabaseException(EINVAL, + Logger::Get().Fatal( + LOGID("Database"), "Wrong database version - Got: " + std::to_string(ver) + " want: " + std::to_string(m_databaseVersion) ); } } +std::unique_ptr Database::FindNntpEntry(std::uint64_t id) +{ + const auto path = GetNewsGroupFilePath(); + if (!std::filesystem::exists(path)) return nullptr; + SerializableFile io; + io.Open(path); + CheckDbVersion(io); + const std::uint64_t numGroups = io.ReadInt64(); + std::unique_ptr result = nullptr; + for (std::uint64_t n = 0; n != numGroups; ++n) + { + NntpListEntry entry; + io >> entry; + if (entry.id == id) + { + result = std::make_unique(entry); + break; + } + } + return result; +} + std::unique_ptr Database::FindNntpEntry( const std::string& subject) { @@ -85,8 +100,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID) const auto path = GetNewsGroupFilePath(); if (!std::filesystem::exists(path)) { - throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: " - + std::to_string(newsgroupID)); + Logger::Get().Fatal( + LOGID("Database"), + "No indexed articles for newsgroup: " + + std::to_string(newsgroupID) + ); } SerializableFile io; io.Open(path); @@ -101,8 +119,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID) return entry.lastIndexedArticle; } } - throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: " - + std::to_string(newsgroupID)); + Logger::Get().Fatal( + LOGID("Database"), + "No indexed articles for newsgroup: " + std::to_string(newsgroupID) + ); + return NntpListEntry::NOT_INDEXED; } std::filesystem::path Database::GetTokenFilePath( @@ -185,8 +206,8 @@ void Database::ParseTokenFile( { if (!std::filesystem::exists(dbFile)) { - throw DatabaseException( - ENOTFOUND, + Logger::Get().Fatal( + LOGID("Database"), "File does not exist: " + dbFile.string() ); } @@ -195,6 +216,7 @@ void Database::ParseTokenFile( const std::uint64_t tokenCount = io.ReadInt64(); for (std::uint64_t i = 0; i != tokenCount; ++i) { + if (Application::Get().ShouldStop()) return; ArticleEntry token; io >> token; onParse(token); @@ -221,9 +243,11 @@ void Database::SetLastIndexedArticle( } if (!found) { - throw DatabaseException(EINVAL, + Logger::Get().Fatal( + LOGID("Database"), "Attempt to update newsgroup not found in database - id: " - + std::to_string(newsgroupID)); + + std::to_string(newsgroupID) + ); } UpdateNewsgroupList(*outItems); } @@ -239,7 +263,7 @@ void Database::SaveSearchTokens( " ", m_maxTreeDepth, [&](const std::string& subToken, const std::string& str){ - const std::string tok = m_app.GetFilter().ProcessToken( + const std::string tok = Application::Get().GetFilter().ProcessToken( subToken, str ); @@ -322,11 +346,6 @@ void Database::SaveToken( io << std::uint64_t{1}; } // write out token. -#if 0 - std::cout << "Token: " << subtoken << std::endl; - std::cout << "Saving into file: " << path << std::endl; - std::cout << "Token hash: " << HashBytesToString(token.hash) << std::endl << std::endl; -#endif io << token; } @@ -352,15 +371,6 @@ std::unique_ptr> Database::Search( const auto foundTokens = LoadTokens(path, searchToken); if (foundTokens->empty()) continue; result->insert(result->end(), foundTokens->begin(), foundTokens->end()); - std::cout << std::left << std::setw(searchString.length() + 7) - << "token: " + searchToken - << std::setw(3) << " | " - << std::setw(10) - << "db file: " << path.string() - << std::setw(3) << " | " - << std::setw(9) - << "#results: " + std::to_string(foundTokens->size()) - << std::endl; } return result; } diff --git a/src/Dns.cpp b/src/Dns.cpp index d9fa8e9..6de4764 100644 --- a/src/Dns.cpp +++ b/src/Dns.cpp @@ -17,6 +17,8 @@ #include "usenetsearch/Dns.h" +#include "usenetsearch/Logger.h" + #include #include @@ -64,7 +66,8 @@ std::vector DnsResolve( } else { - throw DnsResolveException(getAddrInfoResult, + Logger::Get().Fatal( + LOGID("Dns"), "Could not resolve host " + host + ": - Error (" + std::to_string(getAddrInfoResult) + ") - " + gai_strerror(getAddrInfoResult) @@ -73,7 +76,8 @@ std::vector DnsResolve( } if (result == nullptr) { - throw DnsResolveException(ETIMEDOUT, + Logger::Get().Fatal( + LOGID("Dns"), "Timed out trying to resolve host: " + host ); } diff --git a/src/Except.cpp b/src/Except.cpp index 8e41e76..a3209cb 100644 --- a/src/Except.cpp +++ b/src/Except.cpp @@ -19,17 +19,11 @@ namespace usenetsearch { - UsenetSearchException::UsenetSearchException(int errorCode, const std::string& message) + UsenetSearchException::UsenetSearchException(const std::string& message) { - m_errorCode = errorCode; m_message = message; } - int UsenetSearchException::Code() const - { - return m_errorCode; - } - const char* UsenetSearchException::what() const noexcept { return m_message.c_str(); diff --git a/src/Filter.cpp b/src/Filter.cpp index 5eec53b..294a021 100644 --- a/src/Filter.cpp +++ b/src/Filter.cpp @@ -36,7 +36,7 @@ void Filter::Init() // Pre-compile regexes for all the subtokens that should be erased. std::for_each(eraseTokens.begin(), eraseTokens.end(), [&](const std::string& tok){ - const std::wstring wtok = WideStringFromString(tok); + const std::wstring wtok = StringToLower(WideStringFromString(tok)); m_eraseTokenRegexes.emplace( std::make_unique(L"^" + wtok + L"\\s+"), std::wstring{} @@ -80,17 +80,15 @@ bool Filter::ProcessNewsgroup(const std::string& newsgroup) const std::string Filter::ProcessSearchString(const std::string& searchString) const { - std::wstring str = WideStringFromString(searchString); + std::wstring str = StringToLower(WideStringFromString(searchString)); std::remove_if(str.begin(), str.end(), [](wchar_t c){ // Remove control characters. if (c < 0x20) return true; // ascii control chars if ((c > 0x7e) && (c < 0xa0)) return true; // utf8 control chars return false; // don't delete anything else }); - // Remove Re: for obvious reasons - str = StringRemove(StringToLower(str), std::wstring{L"re:"}); // Remove punctuation and stuff by converting to whitespace - static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\+\\[\\]\"\\-<>]+"); + static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\[\\]\"\\-<>]+"); str = std::regex_replace(str, rxPunctuation, L" "); // Process erase subtoken list. std::for_each(m_eraseTokenRegexes.begin(), m_eraseTokenRegexes.end(), diff --git a/src/Indexer.cpp b/src/Indexer.cpp index c367cb6..e61af12 100644 --- a/src/Indexer.cpp +++ b/src/Indexer.cpp @@ -17,11 +17,10 @@ #include "usenetsearch/Indexer.h" +#include "usenetsearch/Application.h" #include "usenetsearch/Logger.h" #include "usenetsearch/StringUtils.h" -#include - namespace usenetsearch { // SearchResult class ---------------------------------------------------------- @@ -110,48 +109,46 @@ bool SearchResult::operator<=(const SearchResult& other) const // Indexer class --------------------------------------------------------------- -Indexer::Indexer(Application& app, UsenetClient& client) - : m_app(app), m_client(client) +Indexer::Indexer(UsenetClient& client) + : m_client(client) { - m_threads.MaxThreads(m_app.GetConfig().MaxThreads()); + m_threads.MaxThreads(Application::Get().GetConfig().MaxThreads()); } void Indexer::Connect() { m_client.Connect( - m_app.GetConfig().NNTPServerHost(), - m_app.GetConfig().NNTPServerPort(), - m_app.GetConfig().NNTPServerSSL() + Application::Get().GetConfig().NNTPServerHost(), + Application::Get().GetConfig().NNTPServerPort(), + Application::Get().GetConfig().NNTPServerSSL() ); m_client.Authenticate( - m_conv.from_bytes(m_app.GetConfig().NNTPServerUser()), - m_conv.from_bytes(m_app.GetConfig().NNTPServerPassword()) + m_conv.from_bytes(Application::Get().GetConfig().NNTPServerUser()), + m_conv.from_bytes(Application::Get().GetConfig().NNTPServerPassword()) ); } void Indexer::Index(const std::vector& newsgroups) { - /** - * @todo Replace all stdout stuff with Logger class. - */ - const size_t batchSize = m_app.GetConfig().BatchSize(); + const size_t batchSize = Application::Get().GetConfig().BatchSize(); for (const auto& group: newsgroups) { const std::wstring newsgroup = m_conv.from_bytes(group.name); - std::cout << "Setting group to " << group.name << "..."; - std::cout.flush(); + Logger::Get().Debug(LOGID("Indexer"), "Setting group to " + group.name); m_client.Group(newsgroup); - std::cout << "DONE." << std::endl; - std::cout << "Reading headers in " << group.name << " " - << "(.=" << batchSize << " headers)." << std::endl; - std::cout.flush(); + Logger::Get().Debug( + LOGID("Indexer"), + "Reading headers in " + group.name + " " + + "(batch size = " + std::to_string(batchSize) + " headers)." + ); std::atomic headerCount{0}; const std::atomic groupID = group.id; - std::reference_wrapper dbref = std::ref(m_app.GetDb()); std::uint32_t startMessage = 0; try { - startMessage = dbref.get().GetLastIndexedArticle(groupID); + startMessage = Application::Get().GetDb().GetLastIndexedArticle( + groupID + ); if (startMessage == NntpListEntry::NOT_INDEXED) { startMessage = 0; @@ -165,22 +162,27 @@ void Indexer::Index(const std::vector& newsgroups) { startMessage = 0; } - - std::cout << "Indexing starting at message: " - << std::to_string(startMessage) << std::endl; + Logger::Get().Debug( + LOGID("Indexer"), + "Indexing starting at message: " + + std::to_string(startMessage) + ); + if (Application::Get().ShouldStop()) return; m_client.ProcessHeaders(startMessage, - [this, &startMessage, &headerCount, &dbref, &groupID](std::shared_ptr headers){ - m_threads.Queue([this, headers, &startMessage, &headerCount, &dbref, &groupID](){ + [this, &startMessage, &headerCount, &groupID](std::shared_ptr headers){ + if (Application::Get().ShouldStop()) return; + m_threads.Queue([this, headers, &startMessage, &headerCount, &groupID](){ std::uint64_t lastArticle{0}; for (const auto& header: *headers) { + if (Application::Get().ShouldStop()) return; const std::uint64_t articleID{header.articleID}; std::string subject = header.subject; - subject = m_app.GetFilter().ProcessSearchString( + subject = Application::Get().GetFilter().ProcessSearchString( subject ); if (subject == "") continue; - dbref.get().SaveSearchTokens( + Application::Get().GetDb().SaveSearchTokens( groupID, articleID, subject @@ -195,20 +197,20 @@ void Indexer::Index(const std::vector& newsgroups) // Update last-indexed id for the newsgroup. if (startMessage < lastArticle) { - dbref.get().SetLastIndexedArticle( + Application::Get().GetDb().SetLastIndexedArticle( groupID, lastArticle ); } - std::cout << "."; - std::cout.flush(); + Logger::Get().Debug("Indexer::Index", "Finished batch."); }); }, batchSize ); m_threads.JoinThreads(); - std::cout << "DONE." << std::endl; - std::cout << "Saved " << headerCount << " headers." << std::endl; - std::cout.flush(); + Logger::Get().Debug( + LOGID("Indexer"), + "Saved " + std::to_string(headerCount) + " headers." + ); } } @@ -216,13 +218,21 @@ std::unique_ptr Indexer::Search( const std::string& searchString) { auto result = std::make_unique(); - const std::string sstr = m_app.GetFilter().ProcessSearchString( + const std::string sstr = Application::Get().GetFilter().ProcessSearchString( searchString ); - auto searchResults = m_app.GetDb().Search(sstr); + const auto searchHash = StringHashBytes(sstr); + auto searchResults = Application::Get().GetDb().Search(sstr); if (!searchResults) return result; for(const ArticleEntry& entry: *searchResults) { + if (Application::Get().ShouldStop()) + { + Logger::Get().Fatal( + "Indexer", + "Interrupted." + ); + } SearchResult sr(entry); // Check if a matching entry already exists in the result set, if so, // increment count. Otherwise, append a new entry. @@ -230,6 +240,9 @@ std::unique_ptr Indexer::Search( if (it != result->end()) { (*it).Inc(); + // An exact match gets double points to ensure it's above other + // partial matches. + if (entry.hash == searchHash) (*it).Inc(); } else { diff --git a/src/Logger.cpp b/src/Logger.cpp index a67e57f..4941941 100644 --- a/src/Logger.cpp +++ b/src/Logger.cpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace usenetsearch { @@ -27,7 +28,7 @@ std::unique_ptr Logger::m_instance; Logger::Logger() {} -Logger& Logger::get() +Logger& Logger::Get() { if (m_instance == nullptr) { @@ -36,19 +37,27 @@ Logger& Logger::get() return *m_instance; } -void Logger::debug(const std::string& msg) const +void Logger::Debug(const std::string& func, const std::string& msg) { - std::cout << "[debug] " << msg << std::endl << std::flush; + std::lock_guard lock(m_stdOutLock); + std::cout << "[debug] [" + func + "] " << msg << std::endl << std::flush; } -void Logger::info(const std::string& msg) const +void Logger::Info(const std::string& func, const std::string& msg) { - std::cout << "[info] " << msg << std::endl << std::flush; + std::lock_guard lock(m_stdOutLock); + std::cout << "[info] [" + func + "] " << msg << std::endl << std::flush; } -void Logger::error(const std::string& msg) const +void Logger::Error(const std::string& func, const std::string& msg) { - std::cerr << "[error] " << msg << std::endl << std::flush; + std::lock_guard lock(m_stdErrLock); + std::cerr << "[error] [" + func + "] " << msg << std::endl << std::flush; +} + +std::string LogName(const std::string& module, const std::string& func) +{ + return module + std::string{"::"} + func; } } // namespace usenetsearch diff --git a/src/SSLConnection.cpp b/src/SSLConnection.cpp index 2d5802c..4463841 100644 --- a/src/SSLConnection.cpp +++ b/src/SSLConnection.cpp @@ -17,6 +17,8 @@ #include "usenetsearch/SSLConnection.h" +#include "usenetsearch/Logger.h" + #include #include @@ -38,8 +40,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret) if (result == SSL_ERROR_SYSCALL) { if (errno == 0) return SSLReturnState::SUCCESS; - throw SSLException(errno, std::string{"SSL error: "} - + std::strerror(errno)); + Logger::Get().Fatal( + LOGID("SSLConnection"), + std::string{"SSL error: "} + std::strerror(errno) + ); } if (result == SSL_ERROR_SSL) { @@ -48,8 +52,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret) errorCode, nullptr ); - throw SSLException(errorCode, - "SSL error: " + errorString); + Logger::Get().Fatal( + LOGID("SSLConnection"), + "SSL error: " + errorString + ); } return SSLReturnState::RETRY; } @@ -58,14 +64,16 @@ void SSLConnection::Connect() { if (m_tcpConnection == nullptr) { - throw SSLException(EBADF, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Null tcp connection when attempting ssl connect." ); } int fd = m_tcpConnection->FileDescriptor(); if (!fd) { - throw SSLException(EBADF, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Bad file descriptor (" + std::to_string(fd) + ") when attempting to ssl connect." ); @@ -93,8 +101,8 @@ void SSLConnection::Connect() ); if (timeDelta > m_connectionTimeout) { - throw SSLException( - ETIMEDOUT, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Timed out while trying to establish SSL connection." ); } @@ -115,7 +123,8 @@ std::string SSLConnection::Read(size_t amount) { if (m_sslContext == nullptr) { - throw SSLException(EBADF, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Attempted to write over SSL socket without SSL context." ); } @@ -155,7 +164,8 @@ void SSLConnection::Write(const std::string& data) { if (m_sslContext == nullptr) { - throw SSLException(EBADF, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Attempted to write over SSL socket without SSL context." ); } @@ -184,8 +194,8 @@ void SSLConnection::Write(const std::string& data) (timeDelta > m_ioTimeout) || (m_ioTimeout == std::chrono::milliseconds{0}) ) { - throw SSLException( - ETIMEDOUT, + Logger::Get().Fatal( + LOGID("SSLConnection"), "Timed out while trying to write to SSL connection." ); } diff --git a/src/Serialize.cpp b/src/Serialize.cpp index 4c8f977..8618ebc 100644 --- a/src/Serialize.cpp +++ b/src/Serialize.cpp @@ -15,7 +15,9 @@ #include "usenetsearch/Serialize.h" +#include "usenetsearch/Application.h" #include "usenetsearch/Database.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/ScopeExit.h" #include "usenetsearch/UsenetClient.h" @@ -26,11 +28,13 @@ #include #include +#include #include #include #include #include #include +#include #include namespace usenetsearch { @@ -50,17 +54,34 @@ void SerializableFile::FileLock() { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); } int ret{EWOULDBLOCK}; - while ((ret == EWOULDBLOCK) || (ret == EINTR)) ret = flock(m_fd, LOCK_EX); + while ((ret == EWOULDBLOCK) || (ret == EINTR)) + { + if (Application::Get().ShouldStop()) + { + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Interrupt while trying to establish a lock on file (" + + m_fileName.string() + ")." + ); + } + ret = flock(m_fd, LOCK_EX); + if ((ret == EWOULDBLOCK) || (ret == EINTR)) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } if (ret != 0) { - throw FileIOException( - errno, "Error trying to lock file: " + m_fileName.string() + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Error trying to lock file: " + m_fileName.string() + " : " + std::strerror(errno) ); } @@ -71,7 +92,8 @@ void SerializableFile::FileUnlock() { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -79,8 +101,9 @@ void SerializableFile::FileUnlock() const int ret = flock(m_fd, LOCK_UN); if (ret != 0) { - throw FileIOException( - errno, "Error trying to unlock file: " + m_fileName.string() + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Error trying to unlock file: " + m_fileName.string() + " : " + std::strerror(errno) ); } @@ -108,8 +131,11 @@ void SerializableFile::Open(const std::string& fileName) int ret = open(fileName.c_str(), flags, 0644); if (ret < 0) { - throw FileIOException(errno, "Could not open file: " + fileName - + " : " + std::string{std::strerror(errno)}); + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Could not open file: " + fileName + + " : " + std::string{std::strerror(errno)} + ); } m_fd = ret; if (m_lockOnOpen) FileLock(); @@ -119,7 +145,8 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -128,9 +155,11 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const Seek(offset, std::ios_base::beg); if (lockf(m_fd, F_LOCK, size) == -1) { - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Could not acquire a write lock on file: " + m_fileName.string() - + " : " + std::strerror(errno)); + + " : " + std::strerror(errno) + ); } Seek(pos, std::ios_base::beg); } @@ -139,7 +168,8 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -148,9 +178,11 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const Seek(offset, std::ios_base::beg); if (lockf(m_fd, F_ULOCK, size) == -1) { - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Could not release a write lock on file: " + m_fileName.string() - + " : " + std::strerror(errno)); + + " : " + std::strerror(errno) + ); } Seek(pos, std::ios_base::beg); } @@ -159,7 +191,8 @@ std::string SerializableFile::ReadStr(size_t size) const { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -176,7 +209,8 @@ std::string SerializableFile::ReadStr(size_t size) const const auto readNow = read(m_fd, &result[0], size); if (readNow == -1) { - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Error while reading from file: " + m_fileName.string() + " : " + std::strerror(errno) ); @@ -219,7 +253,8 @@ void SerializableFile::Seek( { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -241,7 +276,8 @@ void SerializableFile::Seek( const auto newOffset = lseek(m_fd, offset, whence); if (newOffset == -1) { - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Could not set cursor position in file: " + m_fileName.string() + " : " + std::strerror(errno) ); @@ -261,7 +297,8 @@ size_t SerializableFile::Tell() const { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -269,7 +306,8 @@ size_t SerializableFile::Tell() const const auto result = lseek(m_fd, 0, SEEK_CUR); if (result == -1) { - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Could not get cursor position in file: " + m_fileName.string() + " : " + std::strerror(errno) ); @@ -301,7 +339,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const { if (!m_fd) { - throw FileIOException(EBADFD, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Attempt to write to a file (" + m_fileName.string() + " ) that isn't open." ); @@ -323,7 +362,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const { if (errno == 0) continue; } - throw FileIOException(errno, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Failure while writing to file: " + m_fileName.string() + " : " + std::strerror(errno) ); @@ -452,7 +492,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj) in >> STX; if ((SOH != 1) || (STX != 2)) { - throw SerializeException(EBADMSG, "Bad magic number in entry header."); + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Bad magic number in entry header." + ); } for (std::uint8_t i = 0; i != 16; ++i) { @@ -464,7 +507,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj) in >> EOT; if ((ETX != 3) || (EOT != 4)) { - throw SerializeException(EBADMSG, "Bad magic number in entry footer."); + Logger::Get().Fatal( + LOGID("SerializableFile"), + "Bad magic number in entry footer." + ); } return in; } @@ -509,7 +555,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj) in >> STX; if ((SOH != 1) || (STX != 2)) { - throw SerializeException(EBADMSG, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Bad magic number in NNTP entry header." ); } @@ -524,7 +571,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj) in >> EOT; if ((ETX != 3) || (EOT != 4)) { - throw SerializeException(EBADMSG, + Logger::Get().Fatal( + LOGID("SerializableFile"), "Bad magic number in NNTP entry footer." ); } diff --git a/src/StringUtils.cpp b/src/StringUtils.cpp index d4d716e..29cbb03 100644 --- a/src/StringUtils.cpp +++ b/src/StringUtils.cpp @@ -17,6 +17,8 @@ #include "usenetsearch/StringUtils.h" +#include "usenetsearch/Logger.h" + #include #include @@ -111,9 +113,11 @@ bool StringToBoolean(const std::string& str) const std::string lstr = StringTrim(StringToLower(str)); if ((lstr == "true") || (lstr == "yes") || (lstr == "1")) return true; if ((lstr == "false") || (lstr == "no") || (lstr == "0")) return false; - throw StringException(EINVAL, + Logger::Get().Fatal( + LOGID("StringUtils"), "The string \"" + str + "\" is not a valid boolean value." ); + return false; } bool StringToBoolean(const std::wstring& str) @@ -122,10 +126,12 @@ bool StringToBoolean(const std::wstring& str) if ((lstr == L"true") || (lstr == L"yes") || (lstr == L"1")) return true; if ((lstr == L"false") || (lstr == L"no") || (lstr == L"0")) return false; std::wstring_convert> conv; - throw StringException(EINVAL, + Logger::Get().Fatal( + LOGID("StringUtils"), "The string \"" + conv.to_bytes(str) + "\" is not a valid boolean value." ); + return false; } void StringTreeOperation( @@ -135,6 +141,7 @@ void StringTreeOperation( std::function Fn) { const auto tokens = StringSplit(searchString, splitBy); + std::vector tokenList; for (auto outerIt = tokens.begin(); outerIt != tokens.end(); outerIt++) { for (size_t depth = 1; depth != maxDepth + 1; ++depth) @@ -142,7 +149,16 @@ void StringTreeOperation( const auto endIt = outerIt + depth; const auto subset = std::vector(outerIt, endIt); const auto subToken = StringJoin(subset, splitBy); - Fn(subToken, searchString); + // Check if we already have this token. + // + // For phrases like "we went here and then we went there" this would + // avoid indexing the tokens 'we' and 'went' and 'we went' twice. + if (std::find(tokenList.begin(), tokenList.end(), subToken) + == tokenList.end()) + { + Fn(subToken, searchString); + tokenList.emplace_back(subToken); + } if (endIt == tokens.end()) break; } } diff --git a/src/TcpConnection.cpp b/src/TcpConnection.cpp index f944628..717dec4 100644 --- a/src/TcpConnection.cpp +++ b/src/TcpConnection.cpp @@ -15,6 +15,7 @@ along with UsenetSearch. If not, see . */ +#include "usenetsearch/Logger.h" #include "usenetsearch/TcpConnection.h" #include "usenetsearch/Dns.h" @@ -45,7 +46,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port) const std::vector addresses = DnsResolve(host, port); if (addresses.empty()) { - throw DnsResolveException(EDESTADDRREQ, + Logger::Get().Fatal( + LOGID("TcpConnection"), "The provided host (" + host + ") did not resolve to an address." ); } @@ -66,8 +68,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port) ); if (timeDelta > m_connectionTimeout) { - throw SocketException( - ETIMEDOUT, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Timed out while trying to connect to " + host + ":" + std::to_string(port) + "." ); @@ -75,7 +77,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port) fd = socket(addr.ai_family, SOCK_STREAM, 0); if (fd < 0) { - throw SocketException(errno, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Failed to create socket - Error (" + std::to_string(errno) + ") - " + std::strerror(errno) ); @@ -101,7 +104,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port) else { close(fd); - throw SocketException(errno, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Failed to connect to " + host + ":" + std::to_string(port) + " - Error (" + std::to_string(errno) + ") - " + strerror(errno) @@ -152,7 +156,8 @@ std::string TcpConnection::Read(size_t amount) } else { - throw SocketException(errno, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Error while reading from TCP socket (" + std::to_string(errno) + ") - " + std::strerror(errno) ); @@ -178,7 +183,8 @@ void TcpConnection::Write(const std::string& data) if ((timeDelta > m_ioTimeout) || m_ioTimeout == std::chrono::milliseconds{0}) { - throw SocketException(ETIMEDOUT, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Timed out writing to TCP socket." ); } @@ -191,7 +197,8 @@ void TcpConnection::Write(const std::string& data) } else { - throw SocketException(errno, + Logger::Get().Fatal( + LOGID("TcpConnection"), "Error writing to tcp socket (" + std::to_string(errno) + ") - " + std::strerror(errno) ); diff --git a/src/UsenetClient.cpp b/src/UsenetClient.cpp index 22549c8..fa3c1de 100644 --- a/src/UsenetClient.cpp +++ b/src/UsenetClient.cpp @@ -19,6 +19,7 @@ #include "usenetsearch/Application.h" #include "usenetsearch/Except.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/StringUtils.h" #include @@ -32,10 +33,6 @@ namespace usenetsearch { // UsenetClient class ---------------------------------------------------------- -UsenetClient::UsenetClient(Application& app): m_app(app) -{ -} - void UsenetClient::Authenticate( const std::wstring& user, const std::wstring& password) @@ -45,8 +42,8 @@ void UsenetClient::Authenticate( auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error authenticating with NNTP server: " + response.message ); @@ -56,8 +53,8 @@ void UsenetClient::Authenticate( response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error authenticating with NNTP server: " + response.message ); @@ -83,8 +80,8 @@ void UsenetClient::Connect( } catch (const UsenetSearchException& e) { - throw UsenetClientException( - e.Code(), + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error while trying to connect to host: " + host + ":" + std::to_string(port) + " - " + e.what() ); @@ -93,8 +90,8 @@ void UsenetClient::Connect( const auto serverHello = ReadLine(); if (IsError(serverHello)) { - throw UsenetClientException( - serverHello.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error received from NNTP server: " + serverHello.message ); @@ -108,8 +105,8 @@ void UsenetClient::Group(const std::wstring& newsgroup) auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error changing group to " + StringFromWideString(newsgroup) + " : " + response.message ); @@ -124,8 +121,8 @@ NntpHeader UsenetClient::Head(std::uint64_t articleID) auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error getting headers for article id " + std::to_string(articleID) + " : " + response.message @@ -164,8 +161,8 @@ std::unique_ptr> UsenetClient::List() const auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Failed to fetch newsgroup list from server, " + std::string{"server responded with: "} + response.message @@ -189,7 +186,7 @@ std::unique_ptr> UsenetClient::List() entry.status = fields[4]; entry.id = 0; // incremented by db when saving. entry.lastIndexedArticle = NntpListEntry::NOT_INDEXED; - if (m_app.GetFilter().ProcessNewsgroup(entry.name)) + if (Application::Get().GetFilter().ProcessNewsgroup(entry.name)) { result->emplace_back(entry); } @@ -202,7 +199,9 @@ std::unique_ptr> UsenetClient::ListGroup( const std::wstring& newsGroup) { auto result = std::make_unique>(); - if (!m_app.GetFilter().ProcessNewsgroup(StringFromWideString(newsGroup))) + if (!Application::Get().GetFilter().ProcessNewsgroup( + StringFromWideString(newsGroup)) + ) { return result; } @@ -212,8 +211,8 @@ std::unique_ptr> UsenetClient::ListGroup( const auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Failed to fetch newsgroup list from server, " + std::string{"server responded with: "} + response.message @@ -241,8 +240,8 @@ void UsenetClient::ProcessHeaders( const auto response = ReadLine(); if (IsError(response)) { - throw UsenetClientException( - response.code, + Logger::Get().Fatal( + LOGID("UsenetClient"), "Error getting headers: " + response.message ); @@ -293,9 +292,11 @@ NntpMessage UsenetClient::ReadLine() line = ReadUntil("\r\n"); if (line.length() < 2) { - throw UsenetSearchException(EPROTONOSUPPORT, + Logger::Get().Fatal( + LOGID("UsenetClient"), "NNTP protocol error - invalid response from server: " - + line); + + line + ); } std::string codeStr = line.substr(0, 3); result.code = std::stoi(codeStr); diff --git a/src/dbdump.cpp b/src/dbdump.cpp index d67a8dd..4466282 100644 --- a/src/dbdump.cpp +++ b/src/dbdump.cpp @@ -25,10 +25,9 @@ using namespace usenetsearch; int main(int argc, char* argv[]) { - Application app; std::string tokenFile{""}; std::string newsgroupFile{""}; - app.AddFileOption( + Application::Get().AddFileOption( 't', "token db file to dump.", [&tokenFile](const std::string& val) @@ -36,7 +35,7 @@ int main(int argc, char* argv[]) tokenFile = val; } ); - app.AddFileOption( + Application::Get().AddFileOption( 'n', "newsgroup file to dump.", [&newsgroupFile](const std::string& val) @@ -44,20 +43,24 @@ int main(int argc, char* argv[]) newsgroupFile = val; } ); - if (!app.Init(argc, argv)) return 1; + if (!Application::Get().Init(argc, argv)) return 1; if (!tokenFile.empty()) { - app.GetDb().ParseTokenFile(tokenFile, [](const ArticleEntry& token){ - std::cout << "Hash: " << HashBytesToString(token.hash) << " | " - << "NewsgroupID: " << token.newsgroupID << " | " - << "ArticleID: " << token.articleID << std::endl; - }); + Application::Get().GetDb().ParseTokenFile( + tokenFile, + [](const ArticleEntry& token){ + std::cout << "Hash: " << HashBytesToString(token.hash) << " | " + << "NewsgroupID: " << token.newsgroupID << " | " + << "ArticleID: " << token.articleID << std::endl; + } + ); } if (!newsgroupFile.empty()) { - const auto groups = app.GetDb().LoadNewsgroupList(); + const auto groups = Application::Get().GetDb().LoadNewsgroupList(); for(const auto& group: *groups) { + if (Application::Get().ShouldStop()) return 1; std::cout << std::left << std::setw(9) << "Id: " + std::to_string(group.id) << std::setw(3) << " | " diff --git a/src/usenetfind.cpp b/src/usenetfind.cpp index a57770b..162ce51 100644 --- a/src/usenetfind.cpp +++ b/src/usenetfind.cpp @@ -1,7 +1,8 @@ #include "usenetsearch/Application.h" -#include "usenetsearch/UsenetClient.h" #include "usenetsearch/Indexer.h" +#include "usenetsearch/StringUtils.h" +#include "usenetsearch/UsenetClient.h" #include @@ -9,28 +10,27 @@ using namespace usenetsearch; int main(int argc, char* argv[]) { - Application app; std::string searchString; - app.AddStringOption('s', "Search string", + Application::Get().AddStringOption('s', "Search string", [&searchString](const std::string& s){ searchString = s; } ); int maxResults{0}; - app.AddIntegerOption('n', "Maximum results", + Application::Get().AddIntegerOption('n', "Maximum results", [&maxResults](int n){ maxResults = n; } ); - if (!app.Init(argc, argv)) return 1; + if (!Application::Get().Init(argc, argv)) return 1; if (searchString.empty()) { std::cerr << "Missing search string." << std::endl; - app.Usage(argv[0]); + Application::Get().Usage(argv[0]); return 1; } - UsenetClient client(app); - Indexer idx(app, client); + UsenetClient client; + Indexer idx(client); std::unique_ptr results = idx.Search( searchString ); @@ -39,15 +39,24 @@ int main(int argc, char* argv[]) std::cout << "Nothing found." << std::endl; return 0; } + idx.Connect(); size_t resultCounter{0}; for (const auto& sr: *results) { - std::cout << std::left - << std::setw(18) << "Newsgroup id: " + std::to_string(sr.NewsgroupId()) - << std::setw(4) << " | " - << std::setw(17) << "Article id: " + std::to_string(sr.ArticleId()) - << std::setw(4) << " | " - << std::setw(10) << "Hits: " + std::to_string(sr.Hits()) + if (Application::Get().ShouldStop()) return 1; + const std::unique_ptr newsGroup + = Application::Get().GetDb().FindNntpEntry(sr.NewsgroupId()); + if (!newsGroup) continue; + client.Group(WideStringFromString(newsGroup->name)); + const NntpHeader headers = client.Head(sr.ArticleId()); + std::cout + << "Score: " + std::to_string(sr.Hits()) + << " | " + << "Newsgroup: " + newsGroup->name + << " | " + << "Article id: " + std::to_string(sr.ArticleId()) + << " | " + << "Subject: " + headers.subject << std::endl; resultCounter++; if ((maxResults > 0) && (resultCounter >= maxResults)) break; diff --git a/src/usenetindexd.cpp b/src/usenetindexd.cpp index 20753d1..423e4eb 100644 --- a/src/usenetindexd.cpp +++ b/src/usenetindexd.cpp @@ -18,6 +18,7 @@ #include "usenetsearch/Application.h" #include "usenetsearch/Except.h" #include "usenetsearch/Indexer.h" +#include "usenetsearch/Logger.h" #include "usenetsearch/UsenetClient.h" #include @@ -26,30 +27,30 @@ using namespace usenetsearch; int main(int argc, char* argv[]) { - Application app; - if (!app.Init(argc, argv)) return 1; + if (!Application::Get().Init(argc, argv)) return 1; - UsenetClient client(app); - Indexer indexer(app, client); - std::cout << "Connecting to newsgroup server..."; + UsenetClient client; + Indexer indexer(client); + Logger::Get().Debug( + LOGID("usenetindexd"), + "Connecting to newsgroup server..." + ); indexer.Connect(); - std::cout << "" << std::endl; + Logger::Get().Debug(LOGID("usenetindexd"), "Connected."); try { - std::cout << "Getting newsgroup list..."; - std::cout.flush(); + Logger::Get().Debug(LOGID("usenetindexd"), "Getting newsgroup list..."); auto list = client.List(); - app.GetDb().UpdateNewsgroupList(*list); - std::cout << "" << std::endl; - std::cout.flush(); - std::cout << "Found " << list->size() << " newsgroups." << std::endl; - std::cout.flush(); + Application::Get().GetDb().UpdateNewsgroupList(*list); + Logger::Get().Debug( + LOGID("usenetindexd"), + "Found " + std::to_string(list->size()) + " newsgroups." + ); indexer.Index(*list); } catch (const UsenetSearchException& e) { - std::cerr << e.what() << std::endl;; return 1; } return 0;