jns
/
UsenetSearch
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Actual search results! Safer cleanup when interrupted. Consistent include order. And a bunch of other polish

This commit is contained in:
John Sennesael 2021-10-20 17:48:43 -05:00
parent f7831afc9d
commit 6b5bdadeb5
28 changed files with 424 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/usenetsearch/Application.h
View File

@ -56,6 +56,8 @@ class Application
std::string m_configFile{"usenetsearch.conf"};
Database m_db;
Filter m_filter;
/// Holds a pointer to the singleton instance.
static std::unique_ptr<Application> m_instance;
void ExecuteCustomOption(
std::shared_ptr<CommandLineOption>&,
@ -93,10 +95,15 @@ public:
std::string defaultValue = ""
);
bool CanRun() const;
/**
* Gets an instance to the application singleton.
*/
static Application& Get();
Configuration& GetConfig();
Database& GetDb();
Filter& GetFilter();
bool Init(int argc, char* argv[]);
bool ShouldStop() const;
void Usage(const std::string& programName);
};

4
include/usenetsearch/Configuration.h
View File

@ -27,8 +27,8 @@ namespace usenetsearch {
struct ConfigurationException: public UsenetSearchException
{
ConfigurationException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
ConfigurationException(const std::string& message):
UsenetSearchException(message){}
virtual ~ConfigurationException() = default;
};

10
include/usenetsearch/Database.h
View File

@ -47,15 +47,14 @@ std::fstream& operator>>(std::fstream& in, ArticleEntry& obj);
struct DatabaseException: public UsenetSearchException
{
DatabaseException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
DatabaseException(const std::string& message):
UsenetSearchException(message){}
virtual ~DatabaseException() = default;
};
class Database
{
Application& m_app;
std::filesystem::path m_databasePath;
std::uint64_t m_databaseVersion{DatabaseVersion};
std::vector<std::filesystem::path> m_lockedFiles;
@ -91,8 +90,9 @@ class Database
public:
explicit Database(Application& app);
~Database();
Database() = default;
~Database() = default;
std::unique_ptr<NntpListEntry> FindNntpEntry(std::uint64_t id);
std::unique_ptr<NntpListEntry> FindNntpEntry(const std::string& subject);
std::uint32_t GetLastIndexedArticle(std::uint64_t newsgroupID);
std::unique_ptr<std::vector<NntpListEntry>> LoadNewsgroupList();

4
include/usenetsearch/Dns.h
View File

@ -29,8 +29,8 @@ namespace usenetsearch {
struct DnsResolveException: public UsenetSearchException
{
DnsResolveException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
DnsResolveException(const std::string& message):
UsenetSearchException(message){}
virtual ~DnsResolveException() = default;
};

7
include/usenetsearch/Except.h
View File

@ -23,15 +23,12 @@ namespace usenetsearch {
class UsenetSearchException: public std::exception
{
int m_errorCode;
std::string m_message;
public:
UsenetSearchException(int errorCode, const std::string& message);
UsenetSearchException(const std::string& message);
virtual ~UsenetSearchException() = default;
int Code() const;
virtual const char* what() const noexcept override;
};

5
include/usenetsearch/Indexer.h
View File

@ -17,7 +17,7 @@
#pragma once
#include "usenetsearch/Application.h"
#include "usenetsearch/Database.h"
#include "usenetsearch/Filter.h"
#include "usenetsearch/UsenetClient.h"
#include "usenetsearch/ThreadPool.h"
@ -61,14 +61,13 @@ typedef std::vector<SearchResult> SearchResults;
class Indexer
{
Application& m_app;
UsenetClient& m_client;
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> m_conv;
ThreadPool m_threads;
public:
Indexer(Application& app, UsenetClient& client);
Indexer(UsenetClient& client);
void Connect();
void Index(const std::vector<NntpListEntry>& newsgroups);

19
include/usenetsearch/Logger.h
View File

@ -20,6 +20,7 @@
#include <string>
#include <type_traits>
#include <memory>
#include <mutex>
#include <stdexcept>
namespace usenetsearch {
@ -32,6 +33,8 @@ class Logger
/// Holds a pointer to the singleton instance.
static std::unique_ptr<Logger> m_instance;
std::mutex m_stdOutLock;
std::mutex m_stdErrLock;
public:
/**
@ -47,28 +50,28 @@ public:
/**
* Gets an instance to the logger.
*/
static Logger& get();
static Logger& Get();
/**
* Logs a debug message.
*
* @param[in] msg Message to be logged.
*/
void debug(const std::string& msg) const;
void Debug(const std::string& func, const std::string& msg);
/**
* Logs an info message.
*
* @param[in] msg Message to be logged.
*/
void info(const std::string& msg) const;
void Info(const std::string& func, const std::string& msg);
/**
* Logs an error message.
*
* @param[in] msg Message to be logged.
*/
void error(const std::string& msg) const;
void Error(const std::string& func, const std::string& msg);
/**
* Logs an error message and throws.
@ -77,14 +80,18 @@ public:
* @param[in] msg Error message to be logged.
*/
template <typename EXCEPTION>
inline void fatal(const std::string& msg) const
inline void Fatal(const std::string& func, const std::string& msg)
{
static_assert(
std::is_base_of<std::exception, EXCEPTION>::value,
"EXCEPTION must be an std::exception or derrived child.");
error(msg);
Error(func, msg);
throw EXCEPTION(msg);
};
};
std::string LogName(const std::string& module, const std::string& func);
#define LOGID(module) LogName(module, __FUNCTION__)
} // namespace usenetsearch

4
include/usenetsearch/SSLConnection.h
View File

@ -28,8 +28,8 @@ namespace usenetsearch {
struct SSLException: public UsenetSearchException
{
SSLException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
SSLException(const std::string& message):
UsenetSearchException(message){}
virtual ~SSLException() = default;
};

9
include/usenetsearch/Serialize.h
View File

@ -33,16 +33,16 @@ struct NntpListEntry;
struct FileIOException: public UsenetSearchException
{
FileIOException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
FileIOException(const std::string& message):
UsenetSearchException(message){}
virtual ~FileIOException() = default;
};
struct SerializeException: public UsenetSearchException
{
SerializeException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
SerializeException(const std::string& message):
UsenetSearchException(message){}
virtual ~SerializeException() = default;
};
@ -125,5 +125,4 @@ SerializableFile& operator>>(SerializableFile& in, NntpHeader& obj);
SerializableFile& operator<<(SerializableFile& out, const NntpListEntry& obj);
SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj);
} // namespace usenetsearch

4
include/usenetsearch/StringUtils.h
View File

@ -29,8 +29,8 @@ namespace usenetsearch {
struct StringException: public UsenetSearchException
{
StringException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
StringException(const std::string& message):
UsenetSearchException(message){}
virtual ~StringException() = default;
};

4
include/usenetsearch/TcpConnection.h
View File

@ -27,8 +27,8 @@ namespace usenetsearch {
struct SocketException: public UsenetSearchException
{
SocketException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
SocketException(const std::string& message):
UsenetSearchException(message){}
virtual ~SocketException() = default;
};

7
include/usenetsearch/UsenetClient.h
View File

@ -36,8 +36,8 @@ class Application;
struct UsenetClientException: public UsenetSearchException
{
UsenetClientException(int errorCode, const std::string& message):
UsenetSearchException(errorCode, message){}
UsenetClientException(const std::string& message):
UsenetSearchException(message){}
virtual ~UsenetClientException() = default;
};
@ -77,7 +77,6 @@ struct NntpListEntry
class UsenetClient
{
Application& m_app;
std::unique_ptr<SSLConnection> m_ssl;
std::unique_ptr<TcpConnection> m_tcp;
bool m_useSSL{false};
@ -89,7 +88,7 @@ class UsenetClient
public:
UsenetClient(Application& app);
UsenetClient() = default;
void Authenticate(const std::wstring& user, const std::wstring& password);

37
src/Application.cpp
View File

@ -17,8 +17,10 @@
#include "usenetsearch/Application.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/StringUtils.h"
#include <csignal>
#include <filesystem>
#include <iostream>
#include <string>
@ -26,6 +28,15 @@
namespace usenetsearch {
sig_atomic_t _signalled_to_stop{0};
void sigHandler(int)
{
_signalled_to_stop = 1;
}
std::unique_ptr<Application> Application::m_instance;
void Application::AddBooleanOption(
char option,
const std::string& help,
@ -88,6 +99,25 @@ void Application::AddStringOption(
m_commandLineArguments.emplace_back(std::move(val));
}
Application& Application::Get()
{
if (m_instance == nullptr)
{
m_instance = std::make_unique<Application>();
}
return *m_instance;
}
bool Application::ShouldStop() const
{
const bool result = _signalled_to_stop == 1;
if (result)
{
Logger::Get().Info(LOGID("Application"), "Been signalled to stop.");
}
return result;
}
void Application::Usage(const std::string& programName)
{
std::cout << "UsenetSearch - usenet search indexer" << std::endl;
@ -147,7 +177,7 @@ void Application::Usage(const std::string& programName)
std::cout << std::endl;
}
Application::Application() : m_db(*this), m_filter(m_config)
Application::Application() : m_filter(m_config)
{
std::cout.setf(std::ios::unitbuf);
}
@ -249,6 +279,11 @@ bool Application::Init(int argc, char* argv[])
{
ParseArgs(argc, argv);
if (!m_canRun) return false;
// Install signal handlers.
void (*prev_handler)(int);
prev_handler = signal(SIGINT, sigHandler);
prev_handler = signal(SIGTERM, sigHandler);
prev_handler = signal(SIGTSTP, sigHandler);
// Read config, setup db
m_config.Open(m_configFile);
m_db.MaxTreeDepth(m_config.MaxTreeDepth());

19
src/Configuration.cpp
View File

@ -17,6 +17,7 @@
#include "usenetsearch/Configuration.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/ScopeExit.h"
#include "usenetsearch/StringUtils.h"
@ -103,7 +104,8 @@ void Configuration::Open(const std::string& filename)
std::ifstream fin(filename.c_str());
if (!fin.is_open())
{
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
"Could not open configuration file: " + filename
);
}
@ -121,7 +123,8 @@ void Configuration::Open(const std::string& filename)
const auto kvp = StringSplit(line, std::string{":"}, 2);
if (kvp.size() != 2)
{
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
std::string("Invalid configuration in ")
+ filename + std::string(" line ")
+ std::to_string(line_nr)
@ -158,7 +161,8 @@ void Configuration::Open(const std::string& filename)
}
catch (const std::regex_error& e)
{
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
std::string("Invalid configuration in ")
+ filename + std::string(" line ")
+ std::to_string(line_nr) + " : Regular expression \""
@ -175,7 +179,8 @@ void Configuration::Open(const std::string& filename)
}
catch (const std::regex_error& e)
{
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
std::string("Invalid configuration in ")
+ filename + std::string(" line ")
+ std::to_string(line_nr) + " : Regular expression \""
@ -228,7 +233,8 @@ void Configuration::Open(const std::string& filename)
catch (const StringException& e)
{
fin.close();
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
std::string("Invalid configuration in ")
+ filename + std::string(" line ")
+ std::to_string(line_nr) + " - " + e.what()
@ -241,7 +247,8 @@ void Configuration::Open(const std::string& filename)
}
else
{
throw ConfigurationException(EINVAL,
Logger::Get().Fatal<ConfigurationException>(
LOGID("Configuration"),
std::string("Invalid configuration in ")
+ filename + std::string(" line ")
+ std::to_string(line_nr)

78
src/Database.cpp
View File

@ -18,13 +18,13 @@
#include "usenetsearch/Database.h"
#include "usenetsearch/Application.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/StringUtils.h"
#include "usenetsearch/UsenetClient.h"
#include "usenetsearch/ScopeExit.h"
#include "usenetsearch/Serialize.h"
#include "usenetsearch/UsenetClient.h"
#include <iomanip>
#include <iostream>
#include <chrono>
#include <filesystem>
#include <fstream>
@ -36,27 +36,42 @@ namespace usenetsearch {
// Database class --------------------------------------------------------------
Database::Database(Application& app): m_app(app)
{
}
Database::~Database()
{
}
void Database::CheckDbVersion(const SerializableFile& f) const
{
f.Seek(0);
const std::uint64_t ver = f.ReadInt64();
if (ver != m_databaseVersion)
{
throw DatabaseException(EINVAL,
Logger::Get().Fatal<DatabaseException>(
LOGID("Database"),
"Wrong database version - Got: " + std::to_string(ver) + " want: "
+ std::to_string(m_databaseVersion)
);
}
}
std::unique_ptr<NntpListEntry> Database::FindNntpEntry(std::uint64_t id)
{
const auto path = GetNewsGroupFilePath();
if (!std::filesystem::exists(path)) return nullptr;
SerializableFile io;
io.Open(path);
CheckDbVersion(io);
const std::uint64_t numGroups = io.ReadInt64();
std::unique_ptr<NntpListEntry> result = nullptr;
for (std::uint64_t n = 0; n != numGroups; ++n)
{
NntpListEntry entry;
io >> entry;
if (entry.id == id)
{
result = std::make_unique<NntpListEntry>(entry);
break;
}
}
return result;
}
std::unique_ptr<NntpListEntry> Database::FindNntpEntry(
const std::string& subject)
{
@ -85,8 +100,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID)
const auto path = GetNewsGroupFilePath();
if (!std::filesystem::exists(path))
{
throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: "
+ std::to_string(newsgroupID));
Logger::Get().Fatal<DatabaseException>(
LOGID("Database"),
"No indexed articles for newsgroup: "
+ std::to_string(newsgroupID)
);
}
SerializableFile io;
io.Open(path);
@ -101,8 +119,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID)
return entry.lastIndexedArticle;
}
}
throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: "
+ std::to_string(newsgroupID));
Logger::Get().Fatal<DatabaseException>(
LOGID("Database"),
"No indexed articles for newsgroup: " + std::to_string(newsgroupID)
);
return NntpListEntry::NOT_INDEXED;
}
std::filesystem::path Database::GetTokenFilePath(
@ -185,8 +206,8 @@ void Database::ParseTokenFile(
{
if (!std::filesystem::exists(dbFile))
{
throw DatabaseException(
ENOTFOUND,
Logger::Get().Fatal<DatabaseException>(
LOGID("Database"),
"File does not exist: " + dbFile.string()
);
}
@ -195,6 +216,7 @@ void Database::ParseTokenFile(
const std::uint64_t tokenCount = io.ReadInt64();
for (std::uint64_t i = 0; i != tokenCount; ++i)
{
if (Application::Get().ShouldStop()) return;
ArticleEntry token;
io >> token;
onParse(token);
@ -221,9 +243,11 @@ void Database::SetLastIndexedArticle(
}
if (!found)
{
throw DatabaseException(EINVAL,
Logger::Get().Fatal<DatabaseException>(
LOGID("Database"),
"Attempt to update newsgroup not found in database - id: "
+ std::to_string(newsgroupID));
+ std::to_string(newsgroupID)
);
}
UpdateNewsgroupList(*outItems);
}
@ -239,7 +263,7 @@ void Database::SaveSearchTokens(
" ",
m_maxTreeDepth,
[&](const std::string& subToken, const std::string& str){
const std::string tok = m_app.GetFilter().ProcessToken(
const std::string tok = Application::Get().GetFilter().ProcessToken(
subToken,
str
);
@ -322,11 +346,6 @@ void Database::SaveToken(
io << std::uint64_t{1};
}
// write out token.
#if 0
std::cout << "Token: " << subtoken << std::endl;
std::cout << "Saving into file: " << path << std::endl;
std::cout << "Token hash: " << HashBytesToString(token.hash) << std::endl << std::endl;
#endif
io << token;
}
@ -352,15 +371,6 @@ std::unique_ptr<std::vector<ArticleEntry>> Database::Search(
const auto foundTokens = LoadTokens(path, searchToken);
if (foundTokens->empty()) continue;
result->insert(result->end(), foundTokens->begin(), foundTokens->end());
std::cout << std::left << std::setw(searchString.length() + 7)
<< "token: " + searchToken
<< std::setw(3) << " | "
<< std::setw(10)
<< "db file: " << path.string()
<< std::setw(3) << " | "
<< std::setw(9)
<< "#results: " + std::to_string(foundTokens->size())
<< std::endl;
}
return result;
}

8
src/Dns.cpp
View File

@ -17,6 +17,8 @@
#include "usenetsearch/Dns.h"
#include "usenetsearch/Logger.h"
#include <cerrno>
#include <thread>
@ -64,7 +66,8 @@ std::vector<struct addrinfo> DnsResolve(
}
else
{
throw DnsResolveException(getAddrInfoResult,
Logger::Get().Fatal<DnsResolveException>(
LOGID("Dns"),
"Could not resolve host " + host + ": - Error ("
+ std::to_string(getAddrInfoResult) + ") - "
+ gai_strerror(getAddrInfoResult)
@ -73,7 +76,8 @@ std::vector<struct addrinfo> DnsResolve(
}
if (result == nullptr)
{
throw DnsResolveException(ETIMEDOUT,
Logger::Get().Fatal<DnsResolveException>(
LOGID("Dns"),
"Timed out trying to resolve host: " + host
);
}

8
src/Except.cpp
View File

@ -19,17 +19,11 @@
namespace usenetsearch {
UsenetSearchException::UsenetSearchException(int errorCode, const std::string& message)
UsenetSearchException::UsenetSearchException(const std::string& message)
{
m_errorCode = errorCode;
m_message = message;
}
int UsenetSearchException::Code() const
{
return m_errorCode;
}
const char* UsenetSearchException::what() const noexcept
{
return m_message.c_str();

8
src/Filter.cpp
View File

@ -36,7 +36,7 @@ void Filter::Init()
// Pre-compile regexes for all the subtokens that should be erased.
std::for_each(eraseTokens.begin(), eraseTokens.end(),
[&](const std::string& tok){
const std::wstring wtok = WideStringFromString(tok);
const std::wstring wtok = StringToLower(WideStringFromString(tok));
m_eraseTokenRegexes.emplace(
std::make_unique<std::wregex>(L"^" + wtok + L"\\s+"),
std::wstring{}
@ -80,17 +80,15 @@ bool Filter::ProcessNewsgroup(const std::string& newsgroup) const
std::string Filter::ProcessSearchString(const std::string& searchString) const
{
std::wstring str = WideStringFromString(searchString);
std::wstring str = StringToLower(WideStringFromString(searchString));
std::remove_if(str.begin(), str.end(), [](wchar_t c){
// Remove control characters.
if (c < 0x20) return true; // ascii control chars
if ((c > 0x7e) && (c < 0xa0)) return true; // utf8 control chars
return false; // don't delete anything else
});
// Remove Re: for obvious reasons
str = StringRemove(StringToLower(str), std::wstring{L"re:"});
// Remove punctuation and stuff by converting to whitespace
static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\+\\[\\]\"\\-<>]+");
static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\[\\]\"\\-<>]+");
str = std::regex_replace(str, rxPunctuation, L" ");
// Process erase subtoken list.
std::for_each(m_eraseTokenRegexes.begin(), m_eraseTokenRegexes.end(),

87
src/Indexer.cpp
View File

@ -17,11 +17,10 @@
#include "usenetsearch/Indexer.h"
#include "usenetsearch/Application.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/StringUtils.h"
#include <iostream>
namespace usenetsearch {
// SearchResult class ----------------------------------------------------------
@ -110,48 +109,46 @@ bool SearchResult::operator<=(const SearchResult& other) const
// Indexer class ---------------------------------------------------------------
Indexer::Indexer(Application& app, UsenetClient& client)
: m_app(app), m_client(client)
Indexer::Indexer(UsenetClient& client)
: m_client(client)
{
m_threads.MaxThreads(m_app.GetConfig().MaxThreads());
m_threads.MaxThreads(Application::Get().GetConfig().MaxThreads());
}
void Indexer::Connect()
{
m_client.Connect(
m_app.GetConfig().NNTPServerHost(),
m_app.GetConfig().NNTPServerPort(),
m_app.GetConfig().NNTPServerSSL()
Application::Get().GetConfig().NNTPServerHost(),
Application::Get().GetConfig().NNTPServerPort(),
Application::Get().GetConfig().NNTPServerSSL()
);
m_client.Authenticate(
m_conv.from_bytes(m_app.GetConfig().NNTPServerUser()),
m_conv.from_bytes(m_app.GetConfig().NNTPServerPassword())
m_conv.from_bytes(Application::Get().GetConfig().NNTPServerUser()),
m_conv.from_bytes(Application::Get().GetConfig().NNTPServerPassword())
);
}
void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
{
/**
* @todo Replace all stdout stuff with Logger class.
*/
const size_t batchSize = m_app.GetConfig().BatchSize();
const size_t batchSize = Application::Get().GetConfig().BatchSize();
for (const auto& group: newsgroups)
{
const std::wstring newsgroup = m_conv.from_bytes(group.name);
std::cout << "Setting group to " << group.name << "...";
std::cout.flush();
Logger::Get().Debug(LOGID("Indexer"), "Setting group to " + group.name);
m_client.Group(newsgroup);
std::cout << "DONE." << std::endl;
std::cout << "Reading headers in " << group.name << " "
<< "(.=" << batchSize << " headers)." << std::endl;
std::cout.flush();
Logger::Get().Debug(
LOGID("Indexer"),
"Reading headers in " + group.name + " "
+ "(batch size = " + std::to_string(batchSize) + " headers)."
);
std::atomic<std::uint64_t> headerCount{0};
const std::atomic<std::uint64_t> groupID = group.id;
std::reference_wrapper<Database> dbref = std::ref(m_app.GetDb());
std::uint32_t startMessage = 0;
try
{
startMessage = dbref.get().GetLastIndexedArticle(groupID);
startMessage = Application::Get().GetDb().GetLastIndexedArticle(
groupID
);
if (startMessage == NntpListEntry::NOT_INDEXED)
{
startMessage = 0;
@ -165,22 +162,27 @@ void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
{
startMessage = 0;
}
std::cout << "Indexing starting at message: "
<< std::to_string(startMessage) << std::endl;
Logger::Get().Debug(
LOGID("Indexer"),
"Indexing starting at message: "
+ std::to_string(startMessage)
);
if (Application::Get().ShouldStop()) return;
m_client.ProcessHeaders(startMessage,
[this, &startMessage, &headerCount, &dbref, &groupID](std::shared_ptr<NntpHeaders> headers){
m_threads.Queue([this, headers, &startMessage, &headerCount, &dbref, &groupID](){
[this, &startMessage, &headerCount, &groupID](std::shared_ptr<NntpHeaders> headers){
if (Application::Get().ShouldStop()) return;
m_threads.Queue([this, headers, &startMessage, &headerCount, &groupID](){
std::uint64_t lastArticle{0};
for (const auto& header: *headers)
{
if (Application::Get().ShouldStop()) return;
const std::uint64_t articleID{header.articleID};
std::string subject = header.subject;
subject = m_app.GetFilter().ProcessSearchString(
subject = Application::Get().GetFilter().ProcessSearchString(
subject
);
if (subject == "") continue;
dbref.get().SaveSearchTokens(
Application::Get().GetDb().SaveSearchTokens(
groupID,
articleID,
subject
@ -195,20 +197,20 @@ void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
// Update last-indexed id for the newsgroup.
if (startMessage < lastArticle)
{
dbref.get().SetLastIndexedArticle(
Application::Get().GetDb().SetLastIndexedArticle(
groupID, lastArticle
);
}
std::cout << ".";
std::cout.flush();
Logger::Get().Debug("Indexer::Index", "Finished batch.");
});
},
batchSize
);
m_threads.JoinThreads();
std::cout << "DONE." << std::endl;
std::cout << "Saved " << headerCount << " headers." << std::endl;
std::cout.flush();
Logger::Get().Debug(
LOGID("Indexer"),
"Saved " + std::to_string(headerCount) + " headers."
);
}
}
@ -216,13 +218,21 @@ std::unique_ptr<SearchResults> Indexer::Search(
const std::string& searchString)
{
auto result = std::make_unique<SearchResults>();
const std::string sstr = m_app.GetFilter().ProcessSearchString(
const std::string sstr = Application::Get().GetFilter().ProcessSearchString(
searchString
);
auto searchResults = m_app.GetDb().Search(sstr);
const auto searchHash = StringHashBytes(sstr);
auto searchResults = Application::Get().GetDb().Search(sstr);
if (!searchResults) return result;
for(const ArticleEntry& entry: *searchResults)
{
if (Application::Get().ShouldStop())
{
Logger::Get().Fatal<UsenetSearchException>(
"Indexer",
"Interrupted."
);
}
SearchResult sr(entry);
// Check if a matching entry already exists in the result set, if so,
// increment count. Otherwise, append a new entry.
@ -230,6 +240,9 @@ std::unique_ptr<SearchResults> Indexer::Search(
if (it != result->end())
{
(*it).Inc();
// An exact match gets double points to ensure it's above other
// partial matches.
if (entry.hash == searchHash) (*it).Inc();
}
else
{

23
src/Logger.cpp
View File

@ -20,6 +20,7 @@
#include <string>
#include <iostream>
#include <memory>
#include <mutex>
namespace usenetsearch {
@ -27,7 +28,7 @@ std::unique_ptr<Logger> Logger::m_instance;
Logger::Logger() {}
Logger& Logger::get()
Logger& Logger::Get()
{
if (m_instance == nullptr)
{
@ -36,19 +37,27 @@ Logger& Logger::get()
return *m_instance;
}
void Logger::debug(const std::string& msg) const
void Logger::Debug(const std::string& func, const std::string& msg)
{
std::cout << "[debug] " << msg << std::endl << std::flush;
std::lock_guard<std::mutex> lock(m_stdOutLock);
std::cout << "[debug] [" + func + "] " << msg << std::endl << std::flush;
}
void Logger::info(const std::string& msg) const
void Logger::Info(const std::string& func, const std::string& msg)
{
std::cout << "[info] " << msg << std::endl << std::flush;
std::lock_guard<std::mutex> lock(m_stdOutLock);
std::cout << "[info] [" + func + "] " << msg << std::endl << std::flush;
}
void Logger::error(const std::string& msg) const
void Logger::Error(const std::string& func, const std::string& msg)
{
std::cerr << "[error] " << msg << std::endl << std::flush;
std::lock_guard<std::mutex> lock(m_stdErrLock);
std::cerr << "[error] [" + func + "] " << msg << std::endl << std::flush;
}
std::string LogName(const std::string& module, const std::string& func)
{
return module + std::string{"::"} + func;
}
} // namespace usenetsearch

34
src/SSLConnection.cpp
View File

@ -17,6 +17,8 @@
#include "usenetsearch/SSLConnection.h"
#include "usenetsearch/Logger.h"
#include <openssl/ssl.h>
#include <openssl/err.h>
@ -38,8 +40,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret)
if (result == SSL_ERROR_SYSCALL)
{
if (errno == 0) return SSLReturnState::SUCCESS;
throw SSLException(errno, std::string{"SSL error: "}
+ std::strerror(errno));
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
std::string{"SSL error: "} + std::strerror(errno)
);
}
if (result == SSL_ERROR_SSL)
{
@ -48,8 +52,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret)
errorCode,
nullptr
);
throw SSLException(errorCode,
"SSL error: " + errorString);
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"SSL error: " + errorString
);
}
return SSLReturnState::RETRY;
}
@ -58,14 +64,16 @@ void SSLConnection::Connect()
{
if (m_tcpConnection == nullptr)
{
throw SSLException(EBADF,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Null tcp connection when attempting ssl connect."
);
}
int fd = m_tcpConnection->FileDescriptor();
if (!fd)
{
throw SSLException(EBADF,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Bad file descriptor (" + std::to_string(fd)
+ ") when attempting to ssl connect."
);
@ -93,8 +101,8 @@ void SSLConnection::Connect()
);
if (timeDelta > m_connectionTimeout)
{
throw SSLException(
ETIMEDOUT,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Timed out while trying to establish SSL connection."
);
}
@ -115,7 +123,8 @@ std::string SSLConnection::Read(size_t amount)
{
if (m_sslContext == nullptr)
{
throw SSLException(EBADF,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Attempted to write over SSL socket without SSL context."
);
}
@ -155,7 +164,8 @@ void SSLConnection::Write(const std::string& data)
{
if (m_sslContext == nullptr)
{
throw SSLException(EBADF,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Attempted to write over SSL socket without SSL context."
);
}
@ -184,8 +194,8 @@ void SSLConnection::Write(const std::string& data)
(timeDelta > m_ioTimeout)
|| (m_ioTimeout == std::chrono::milliseconds{0}) )
{
throw SSLException(
ETIMEDOUT,
Logger::Get().Fatal<SSLException>(
LOGID("SSLConnection"),
"Timed out while trying to write to SSL connection."
);
}

102
src/Serialize.cpp
View File

@ -15,7 +15,9 @@
#include "usenetsearch/Serialize.h"
#include "usenetsearch/Application.h"
#include "usenetsearch/Database.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/ScopeExit.h"
#include "usenetsearch/UsenetClient.h"
@ -26,11 +28,13 @@
#include <unistd.h>
#include <cerrno>
#include <chrono>
#include <codecvt>
#include <cstdint>
#include <cstring>
#include <locale>
#include <fstream>
#include <thread>
#include <vector>
namespace usenetsearch {
@ -50,17 +54,34 @@ void SerializableFile::FileLock()
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
}
int ret{EWOULDBLOCK};
while ((ret == EWOULDBLOCK) || (ret == EINTR)) ret = flock(m_fd, LOCK_EX);
while ((ret == EWOULDBLOCK) || (ret == EINTR))
{
if (Application::Get().ShouldStop())
{
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Interrupt while trying to establish a lock on file ("
+ m_fileName.string() + ")."
);
}
ret = flock(m_fd, LOCK_EX);
if ((ret == EWOULDBLOCK) || (ret == EINTR))
{
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
}
if (ret != 0)
{
throw FileIOException(
errno, "Error trying to lock file: " + m_fileName.string()
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Error trying to lock file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
}
@ -71,7 +92,8 @@ void SerializableFile::FileUnlock()
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -79,8 +101,9 @@ void SerializableFile::FileUnlock()
const int ret = flock(m_fd, LOCK_UN);
if (ret != 0)
{
throw FileIOException(
errno, "Error trying to unlock file: " + m_fileName.string()
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Error trying to unlock file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
}
@ -108,8 +131,11 @@ void SerializableFile::Open(const std::string& fileName)
int ret = open(fileName.c_str(), flags, 0644);
if (ret < 0)
{
throw FileIOException(errno, "Could not open file: " + fileName
+ " : " + std::string{std::strerror(errno)});
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Could not open file: " + fileName
+ " : " + std::string{std::strerror(errno)}
);
}
m_fd = ret;
if (m_lockOnOpen) FileLock();
@ -119,7 +145,8 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -128,9 +155,11 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const
Seek(offset, std::ios_base::beg);
if (lockf(m_fd, F_LOCK, size) == -1)
{
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Could not acquire a write lock on file: " + m_fileName.string()
+ " : " + std::strerror(errno));
+ " : " + std::strerror(errno)
);
}
Seek(pos, std::ios_base::beg);
}
@ -139,7 +168,8 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -148,9 +178,11 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const
Seek(offset, std::ios_base::beg);
if (lockf(m_fd, F_ULOCK, size) == -1)
{
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Could not release a write lock on file: " + m_fileName.string()
+ " : " + std::strerror(errno));
+ " : " + std::strerror(errno)
);
}
Seek(pos, std::ios_base::beg);
}
@ -159,7 +191,8 @@ std::string SerializableFile::ReadStr(size_t size) const
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -176,7 +209,8 @@ std::string SerializableFile::ReadStr(size_t size) const
const auto readNow = read(m_fd, &result[0], size);
if (readNow == -1)
{
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Error while reading from file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
@ -219,7 +253,8 @@ void SerializableFile::Seek(
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -241,7 +276,8 @@ void SerializableFile::Seek(
const auto newOffset = lseek(m_fd, offset, whence);
if (newOffset == -1)
{
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Could not set cursor position in file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
@ -261,7 +297,8 @@ size_t SerializableFile::Tell() const
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -269,7 +306,8 @@ size_t SerializableFile::Tell() const
const auto result = lseek(m_fd, 0, SEEK_CUR);
if (result == -1)
{
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Could not get cursor position in file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
@ -301,7 +339,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const
{
if (!m_fd)
{
throw FileIOException(EBADFD,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Attempt to write to a file (" + m_fileName.string()
+ " ) that isn't open."
);
@ -323,7 +362,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const
{
if (errno == 0) continue;
}
throw FileIOException(errno,
Logger::Get().Fatal<FileIOException>(
LOGID("SerializableFile"),
"Failure while writing to file: " + m_fileName.string()
+ " : " + std::strerror(errno)
);
@ -452,7 +492,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj)
in >> STX;
if ((SOH != 1) || (STX != 2))
{
throw SerializeException(EBADMSG, "Bad magic number in entry header.");
Logger::Get().Fatal<SerializeException>(
LOGID("SerializableFile"),
"Bad magic number in entry header."
);
}
for (std::uint8_t i = 0; i != 16; ++i)
{
@ -464,7 +507,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj)
in >> EOT;
if ((ETX != 3) || (EOT != 4))
{
throw SerializeException(EBADMSG, "Bad magic number in entry footer.");
Logger::Get().Fatal<SerializeException>(
LOGID("SerializableFile"),
"Bad magic number in entry footer."
);
}
return in;
}
@ -509,7 +555,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj)
in >> STX;
if ((SOH != 1) || (STX != 2))
{
throw SerializeException(EBADMSG,
Logger::Get().Fatal<SerializeException>(
LOGID("SerializableFile"),
"Bad magic number in NNTP entry header."
);
}
@ -524,7 +571,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj)
in >> EOT;
if ((ETX != 3) || (EOT != 4))
{
throw SerializeException(EBADMSG,
Logger::Get().Fatal<SerializeException>(
LOGID("SerializableFile"),
"Bad magic number in NNTP entry footer."
);
}

22
src/StringUtils.cpp
View File

@ -17,6 +17,8 @@
#include "usenetsearch/StringUtils.h"
#include "usenetsearch/Logger.h"
#include <openssl/md5.h>
#include <algorithm>
@ -111,9 +113,11 @@ bool StringToBoolean(const std::string& str)
const std::string lstr = StringTrim(StringToLower(str));
if ((lstr == "true") || (lstr == "yes") || (lstr == "1")) return true;
if ((lstr == "false") || (lstr == "no") || (lstr == "0")) return false;
throw StringException(EINVAL,
Logger::Get().Fatal<StringException>(
LOGID("StringUtils"),
"The string \"" + str + "\" is not a valid boolean value."
);
return false;
}
bool StringToBoolean(const std::wstring& str)
@ -122,10 +126,12 @@ bool StringToBoolean(const std::wstring& str)
if ((lstr == L"true") || (lstr == L"yes") || (lstr == L"1")) return true;
if ((lstr == L"false") || (lstr == L"no") || (lstr == L"0")) return false;
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv;
throw StringException(EINVAL,
Logger::Get().Fatal<StringException>(
LOGID("StringUtils"),
"The string \"" + conv.to_bytes(str)
+ "\" is not a valid boolean value."
);
return false;
}
void StringTreeOperation(
@ -135,6 +141,7 @@ void StringTreeOperation(
std::function<void(const std::string& subToken, const std::string& str)> Fn)
{
const auto tokens = StringSplit(searchString, splitBy);
std::vector<std::string> tokenList;
for (auto outerIt = tokens.begin(); outerIt != tokens.end(); outerIt++)
{
for (size_t depth = 1; depth != maxDepth + 1; ++depth)
@ -142,7 +149,16 @@ void StringTreeOperation(
const auto endIt = outerIt + depth;
const auto subset = std::vector<std::string>(outerIt, endIt);
const auto subToken = StringJoin(subset, splitBy);
Fn(subToken, searchString);
// Check if we already have this token.
//
// For phrases like "we went here and then we went there" this would
// avoid indexing the tokens 'we' and 'went' and 'we went' twice.
if (std::find(tokenList.begin(), tokenList.end(), subToken)
== tokenList.end())
{
Fn(subToken, searchString);
tokenList.emplace_back(subToken);
}
if (endIt == tokens.end()) break;
}
}

23
src/TcpConnection.cpp
View File

@ -15,6 +15,7 @@
along with UsenetSearch. If not, see <https://www.gnu.org/licenses/>.
*/
#include "usenetsearch/Logger.h"
#include "usenetsearch/TcpConnection.h"
#include "usenetsearch/Dns.h"
@ -45,7 +46,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
const std::vector<struct addrinfo> addresses = DnsResolve(host, port);
if (addresses.empty())
{
throw DnsResolveException(EDESTADDRREQ,
Logger::Get().Fatal<DnsResolveException>(
LOGID("TcpConnection"),
"The provided host (" + host + ") did not resolve to an address."
);
}
@ -66,8 +68,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
);
if (timeDelta > m_connectionTimeout)
{
throw SocketException(
ETIMEDOUT,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Timed out while trying to connect to " + host + ":"
+ std::to_string(port) + "."
);
@ -75,7 +77,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
fd = socket(addr.ai_family, SOCK_STREAM, 0);
if (fd < 0)
{
throw SocketException(errno,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Failed to create socket - Error (" + std::to_string(errno)
+ ") - " + std::strerror(errno)
);
@ -101,7 +104,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
else
{
close(fd);
throw SocketException(errno,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Failed to connect to " + host + ":"
+ std::to_string(port) + " - Error ("
+ std::to_string(errno) + ") - " + strerror(errno)
@ -152,7 +156,8 @@ std::string TcpConnection::Read(size_t amount)
}
else
{
throw SocketException(errno,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Error while reading from TCP socket (" + std::to_string(errno)
+ ") - " + std::strerror(errno)
);
@ -178,7 +183,8 @@ void TcpConnection::Write(const std::string& data)
if ((timeDelta > m_ioTimeout)
|| m_ioTimeout == std::chrono::milliseconds{0})
{
throw SocketException(ETIMEDOUT,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Timed out writing to TCP socket."
);
}
@ -191,7 +197,8 @@ void TcpConnection::Write(const std::string& data)
}
else
{
throw SocketException(errno,
Logger::Get().Fatal<SocketException>(
LOGID("TcpConnection"),
"Error writing to tcp socket (" + std::to_string(errno)
+ ") - " + std::strerror(errno)
);

53
src/UsenetClient.cpp
View File

@ -19,6 +19,7 @@
#include "usenetsearch/Application.h"
#include "usenetsearch/Except.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/StringUtils.h"
#include <codecvt>
@ -32,10 +33,6 @@ namespace usenetsearch {
// UsenetClient class ----------------------------------------------------------
UsenetClient::UsenetClient(Application& app): m_app(app)
{
}
void UsenetClient::Authenticate(
const std::wstring& user,
const std::wstring& password)
@ -45,8 +42,8 @@ void UsenetClient::Authenticate(
auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error authenticating with NNTP server: "
+ response.message
);
@ -56,8 +53,8 @@ void UsenetClient::Authenticate(
response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error authenticating with NNTP server: "
+ response.message
);
@ -83,8 +80,8 @@ void UsenetClient::Connect(
}
catch (const UsenetSearchException& e)
{
throw UsenetClientException(
e.Code(),
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error while trying to connect to host: " + host + ":"
+ std::to_string(port) + " - " + e.what()
);
@ -93,8 +90,8 @@ void UsenetClient::Connect(
const auto serverHello = ReadLine();
if (IsError(serverHello))
{
throw UsenetClientException(
serverHello.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error received from NNTP server: "
+ serverHello.message
);
@ -108,8 +105,8 @@ void UsenetClient::Group(const std::wstring& newsgroup)
auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error changing group to " + StringFromWideString(newsgroup) + " : "
+ response.message
);
@ -124,8 +121,8 @@ NntpHeader UsenetClient::Head(std::uint64_t articleID)
auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error getting headers for article id "
+ std::to_string(articleID)
+ " : " + response.message
@ -164,8 +161,8 @@ std::unique_ptr<std::vector<NntpListEntry>> UsenetClient::List()
const auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Failed to fetch newsgroup list from server, "
+ std::string{"server responded with: "}
+ response.message
@ -189,7 +186,7 @@ std::unique_ptr<std::vector<NntpListEntry>> UsenetClient::List()
entry.status = fields[4];
entry.id = 0; // incremented by db when saving.
entry.lastIndexedArticle = NntpListEntry::NOT_INDEXED;
if (m_app.GetFilter().ProcessNewsgroup(entry.name))
if (Application::Get().GetFilter().ProcessNewsgroup(entry.name))
{
result->emplace_back(entry);
}
@ -202,7 +199,9 @@ std::unique_ptr<std::vector<std::uint64_t>> UsenetClient::ListGroup(
const std::wstring& newsGroup)
{
auto result = std::make_unique<std::vector<std::uint64_t>>();
if (!m_app.GetFilter().ProcessNewsgroup(StringFromWideString(newsGroup)))
if (!Application::Get().GetFilter().ProcessNewsgroup(
StringFromWideString(newsGroup))
)
{
return result;
}
@ -212,8 +211,8 @@ std::unique_ptr<std::vector<std::uint64_t>> UsenetClient::ListGroup(
const auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Failed to fetch newsgroup list from server, "
+ std::string{"server responded with: "}
+ response.message
@ -241,8 +240,8 @@ void UsenetClient::ProcessHeaders(
const auto response = ReadLine();
if (IsError(response))
{
throw UsenetClientException(
response.code,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"Error getting headers: "
+ response.message
);
@ -293,9 +292,11 @@ NntpMessage UsenetClient::ReadLine()
line = ReadUntil("\r\n");
if (line.length() < 2)
{
throw UsenetSearchException(EPROTONOSUPPORT,
Logger::Get().Fatal<UsenetClientException>(
LOGID("UsenetClient"),
"NNTP protocol error - invalid response from server: "
+ line);
+ line
);
}
std::string codeStr = line.substr(0, 3);
result.code = std::stoi(codeStr);

23
src/dbdump.cpp
View File

@ -25,10 +25,9 @@ using namespace usenetsearch;
int main(int argc, char* argv[])
{
Application app;
std::string tokenFile{""};
std::string newsgroupFile{""};
app.AddFileOption(
Application::Get().AddFileOption(
't',
"token db file to dump.",
[&tokenFile](const std::string& val)
@ -36,7 +35,7 @@ int main(int argc, char* argv[])
tokenFile = val;
}
);
app.AddFileOption(
Application::Get().AddFileOption(
'n',
"newsgroup file to dump.",
[&newsgroupFile](const std::string& val)
@ -44,20 +43,24 @@ int main(int argc, char* argv[])
newsgroupFile = val;
}
);
if (!app.Init(argc, argv)) return 1;
if (!Application::Get().Init(argc, argv)) return 1;
if (!tokenFile.empty())
{
app.GetDb().ParseTokenFile(tokenFile, [](const ArticleEntry& token){
std::cout << "Hash: " << HashBytesToString(token.hash) << " | "
<< "NewsgroupID: " << token.newsgroupID << " | "
<< "ArticleID: " << token.articleID << std::endl;
});
Application::Get().GetDb().ParseTokenFile(
tokenFile,
[](const ArticleEntry& token){
std::cout << "Hash: " << HashBytesToString(token.hash) << " | "
<< "NewsgroupID: " << token.newsgroupID << " | "
<< "ArticleID: " << token.articleID << std::endl;
}
);
}
if (!newsgroupFile.empty())
{
const auto groups = app.GetDb().LoadNewsgroupList();
const auto groups = Application::Get().GetDb().LoadNewsgroupList();
for(const auto& group: *groups)
{
if (Application::Get().ShouldStop()) return 1;
std::cout << std::left
<< std::setw(9) << "Id: " + std::to_string(group.id)
<< std::setw(3) << " | "

37
src/usenetfind.cpp
View File

@ -1,7 +1,8 @@
#include "usenetsearch/Application.h"
#include "usenetsearch/UsenetClient.h"
#include "usenetsearch/Indexer.h"
#include "usenetsearch/StringUtils.h"
#include "usenetsearch/UsenetClient.h"
#include <iostream>
@ -9,28 +10,27 @@ using namespace usenetsearch;
int main(int argc, char* argv[])
{
Application app;
std::string searchString;
app.AddStringOption('s', "Search string",
Application::Get().AddStringOption('s', "Search string",
[&searchString](const std::string& s){
searchString = s;
}
);
int maxResults{0};
app.AddIntegerOption('n', "Maximum results",
Application::Get().AddIntegerOption('n', "Maximum results",
[&maxResults](int n){
maxResults = n;
}
);
if (!app.Init(argc, argv)) return 1;
if (!Application::Get().Init(argc, argv)) return 1;
if (searchString.empty())
{
std::cerr << "Missing search string." << std::endl;
app.Usage(argv[0]);
Application::Get().Usage(argv[0]);
return 1;
}
UsenetClient client(app);
Indexer idx(app, client);
UsenetClient client;
Indexer idx(client);
std::unique_ptr<SearchResults> results = idx.Search(
searchString
);
@ -39,15 +39,24 @@ int main(int argc, char* argv[])
std::cout << "Nothing found." << std::endl;
return 0;
}
idx.Connect();
size_t resultCounter{0};
for (const auto& sr: *results)
{
std::cout << std::left
<< std::setw(18) << "Newsgroup id: " + std::to_string(sr.NewsgroupId())
<< std::setw(4) << " | "
<< std::setw(17) << "Article id: " + std::to_string(sr.ArticleId())
<< std::setw(4) << " | "
<< std::setw(10) << "Hits: " + std::to_string(sr.Hits())
if (Application::Get().ShouldStop()) return 1;
const std::unique_ptr<NntpListEntry> newsGroup
= Application::Get().GetDb().FindNntpEntry(sr.NewsgroupId());
if (!newsGroup) continue;
client.Group(WideStringFromString(newsGroup->name));
const NntpHeader headers = client.Head(sr.ArticleId());
std::cout
<< "Score: " + std::to_string(sr.Hits())
<< " | "
<< "Newsgroup: " + newsGroup->name
<< " | "
<< "Article id: " + std::to_string(sr.ArticleId())
<< " | "
<< "Subject: " + headers.subject
<< std::endl;
resultCounter++;
if ((maxResults > 0) && (resultCounter >= maxResults)) break;

29
src/usenetindexd.cpp
View File

@ -18,6 +18,7 @@
#include "usenetsearch/Application.h"
#include "usenetsearch/Except.h"
#include "usenetsearch/Indexer.h"
#include "usenetsearch/Logger.h"
#include "usenetsearch/UsenetClient.h"
#include <iostream>
@ -26,30 +27,30 @@ using namespace usenetsearch;
int main(int argc, char* argv[])
{
Application app;
if (!app.Init(argc, argv)) return 1;
if (!Application::Get().Init(argc, argv)) return 1;
UsenetClient client(app);
Indexer indexer(app, client);
std::cout << "Connecting to newsgroup server...";
UsenetClient client;
Indexer indexer(client);
Logger::Get().Debug(
LOGID("usenetindexd"),
"Connecting to newsgroup server..."
);
indexer.Connect();
std::cout << "<OK>" << std::endl;
Logger::Get().Debug(LOGID("usenetindexd"), "Connected.");
try
{
std::cout << "Getting newsgroup list...";
std::cout.flush();
Logger::Get().Debug(LOGID("usenetindexd"), "Getting newsgroup list...");
auto list = client.List();
app.GetDb().UpdateNewsgroupList(*list);
std::cout << "<DONE>" << std::endl;
std::cout.flush();
std::cout << "Found " << list->size() << " newsgroups." << std::endl;
std::cout.flush();
Application::Get().GetDb().UpdateNewsgroupList(*list);
Logger::Get().Debug(
LOGID("usenetindexd"),
"Found " + std::to_string(list->size()) + " newsgroups."
);
indexer.Index(*list);
}
catch (const UsenetSearchException& e)
{
std::cerr << e.what() << std::endl;;
return 1;
}
return 0;