Actual search results! Safer cleanup when interrupted. Consistent include order. And a bunch of other polish
This commit is contained in:
parent
f7831afc9d
commit
6b5bdadeb5
|
@ -56,6 +56,8 @@ class Application
|
|||
std::string m_configFile{"usenetsearch.conf"};
|
||||
Database m_db;
|
||||
Filter m_filter;
|
||||
/// Holds a pointer to the singleton instance.
|
||||
static std::unique_ptr<Application> m_instance;
|
||||
|
||||
void ExecuteCustomOption(
|
||||
std::shared_ptr<CommandLineOption>&,
|
||||
|
@ -93,10 +95,15 @@ public:
|
|||
std::string defaultValue = ""
|
||||
);
|
||||
bool CanRun() const;
|
||||
/**
|
||||
* Gets an instance to the application singleton.
|
||||
*/
|
||||
static Application& Get();
|
||||
Configuration& GetConfig();
|
||||
Database& GetDb();
|
||||
Filter& GetFilter();
|
||||
bool Init(int argc, char* argv[]);
|
||||
bool ShouldStop() const;
|
||||
void Usage(const std::string& programName);
|
||||
};
|
||||
|
||||
|
|
|
@ -27,8 +27,8 @@ namespace usenetsearch {
|
|||
|
||||
struct ConfigurationException: public UsenetSearchException
|
||||
{
|
||||
ConfigurationException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
ConfigurationException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~ConfigurationException() = default;
|
||||
};
|
||||
|
|
|
@ -47,15 +47,14 @@ std::fstream& operator>>(std::fstream& in, ArticleEntry& obj);
|
|||
|
||||
struct DatabaseException: public UsenetSearchException
|
||||
{
|
||||
DatabaseException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
DatabaseException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~DatabaseException() = default;
|
||||
};
|
||||
|
||||
class Database
|
||||
{
|
||||
Application& m_app;
|
||||
std::filesystem::path m_databasePath;
|
||||
std::uint64_t m_databaseVersion{DatabaseVersion};
|
||||
std::vector<std::filesystem::path> m_lockedFiles;
|
||||
|
@ -91,8 +90,9 @@ class Database
|
|||
|
||||
public:
|
||||
|
||||
explicit Database(Application& app);
|
||||
~Database();
|
||||
Database() = default;
|
||||
~Database() = default;
|
||||
std::unique_ptr<NntpListEntry> FindNntpEntry(std::uint64_t id);
|
||||
std::unique_ptr<NntpListEntry> FindNntpEntry(const std::string& subject);
|
||||
std::uint32_t GetLastIndexedArticle(std::uint64_t newsgroupID);
|
||||
std::unique_ptr<std::vector<NntpListEntry>> LoadNewsgroupList();
|
||||
|
|
|
@ -29,8 +29,8 @@ namespace usenetsearch {
|
|||
|
||||
struct DnsResolveException: public UsenetSearchException
|
||||
{
|
||||
DnsResolveException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
DnsResolveException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~DnsResolveException() = default;
|
||||
};
|
||||
|
|
|
@ -23,15 +23,12 @@ namespace usenetsearch {
|
|||
|
||||
class UsenetSearchException: public std::exception
|
||||
{
|
||||
int m_errorCode;
|
||||
std::string m_message;
|
||||
public:
|
||||
|
||||
UsenetSearchException(int errorCode, const std::string& message);
|
||||
UsenetSearchException(const std::string& message);
|
||||
|
||||
virtual ~UsenetSearchException() = default;
|
||||
|
||||
int Code() const;
|
||||
|
||||
virtual const char* what() const noexcept override;
|
||||
|
||||
};
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Database.h"
|
||||
#include "usenetsearch/Filter.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
#include "usenetsearch/ThreadPool.h"
|
||||
|
@ -61,14 +61,13 @@ typedef std::vector<SearchResult> SearchResults;
|
|||
|
||||
class Indexer
|
||||
{
|
||||
Application& m_app;
|
||||
UsenetClient& m_client;
|
||||
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> m_conv;
|
||||
ThreadPool m_threads;
|
||||
|
||||
public:
|
||||
|
||||
Indexer(Application& app, UsenetClient& client);
|
||||
Indexer(UsenetClient& client);
|
||||
|
||||
void Connect();
|
||||
void Index(const std::vector<NntpListEntry>& newsgroups);
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace usenetsearch {
|
||||
|
@ -32,6 +33,8 @@ class Logger
|
|||
|
||||
/// Holds a pointer to the singleton instance.
|
||||
static std::unique_ptr<Logger> m_instance;
|
||||
std::mutex m_stdOutLock;
|
||||
std::mutex m_stdErrLock;
|
||||
|
||||
public:
|
||||
/**
|
||||
|
@ -47,28 +50,28 @@ public:
|
|||
/**
|
||||
* Gets an instance to the logger.
|
||||
*/
|
||||
static Logger& get();
|
||||
static Logger& Get();
|
||||
|
||||
/**
|
||||
* Logs a debug message.
|
||||
*
|
||||
* @param[in] msg Message to be logged.
|
||||
*/
|
||||
void debug(const std::string& msg) const;
|
||||
void Debug(const std::string& func, const std::string& msg);
|
||||
|
||||
/**
|
||||
* Logs an info message.
|
||||
*
|
||||
* @param[in] msg Message to be logged.
|
||||
*/
|
||||
void info(const std::string& msg) const;
|
||||
void Info(const std::string& func, const std::string& msg);
|
||||
|
||||
/**
|
||||
* Logs an error message.
|
||||
*
|
||||
* @param[in] msg Message to be logged.
|
||||
*/
|
||||
void error(const std::string& msg) const;
|
||||
void Error(const std::string& func, const std::string& msg);
|
||||
|
||||
/**
|
||||
* Logs an error message and throws.
|
||||
|
@ -77,14 +80,18 @@ public:
|
|||
* @param[in] msg Error message to be logged.
|
||||
*/
|
||||
template <typename EXCEPTION>
|
||||
inline void fatal(const std::string& msg) const
|
||||
inline void Fatal(const std::string& func, const std::string& msg)
|
||||
{
|
||||
static_assert(
|
||||
std::is_base_of<std::exception, EXCEPTION>::value,
|
||||
"EXCEPTION must be an std::exception or derrived child.");
|
||||
error(msg);
|
||||
Error(func, msg);
|
||||
throw EXCEPTION(msg);
|
||||
};
|
||||
};
|
||||
|
||||
std::string LogName(const std::string& module, const std::string& func);
|
||||
|
||||
#define LOGID(module) LogName(module, __FUNCTION__)
|
||||
|
||||
} // namespace usenetsearch
|
||||
|
|
|
@ -28,8 +28,8 @@ namespace usenetsearch {
|
|||
|
||||
struct SSLException: public UsenetSearchException
|
||||
{
|
||||
SSLException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
SSLException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~SSLException() = default;
|
||||
};
|
||||
|
|
|
@ -33,16 +33,16 @@ struct NntpListEntry;
|
|||
|
||||
struct FileIOException: public UsenetSearchException
|
||||
{
|
||||
FileIOException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
FileIOException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~FileIOException() = default;
|
||||
};
|
||||
|
||||
struct SerializeException: public UsenetSearchException
|
||||
{
|
||||
SerializeException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
SerializeException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~SerializeException() = default;
|
||||
};
|
||||
|
@ -125,5 +125,4 @@ SerializableFile& operator>>(SerializableFile& in, NntpHeader& obj);
|
|||
SerializableFile& operator<<(SerializableFile& out, const NntpListEntry& obj);
|
||||
SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj);
|
||||
|
||||
|
||||
} // namespace usenetsearch
|
||||
|
|
|
@ -29,8 +29,8 @@ namespace usenetsearch {
|
|||
|
||||
struct StringException: public UsenetSearchException
|
||||
{
|
||||
StringException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
StringException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~StringException() = default;
|
||||
};
|
||||
|
|
|
@ -27,8 +27,8 @@ namespace usenetsearch {
|
|||
|
||||
struct SocketException: public UsenetSearchException
|
||||
{
|
||||
SocketException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
SocketException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~SocketException() = default;
|
||||
};
|
||||
|
|
|
@ -36,8 +36,8 @@ class Application;
|
|||
|
||||
struct UsenetClientException: public UsenetSearchException
|
||||
{
|
||||
UsenetClientException(int errorCode, const std::string& message):
|
||||
UsenetSearchException(errorCode, message){}
|
||||
UsenetClientException(const std::string& message):
|
||||
UsenetSearchException(message){}
|
||||
|
||||
virtual ~UsenetClientException() = default;
|
||||
};
|
||||
|
@ -77,7 +77,6 @@ struct NntpListEntry
|
|||
class UsenetClient
|
||||
{
|
||||
|
||||
Application& m_app;
|
||||
std::unique_ptr<SSLConnection> m_ssl;
|
||||
std::unique_ptr<TcpConnection> m_tcp;
|
||||
bool m_useSSL{false};
|
||||
|
@ -89,7 +88,7 @@ class UsenetClient
|
|||
|
||||
public:
|
||||
|
||||
UsenetClient(Application& app);
|
||||
UsenetClient() = default;
|
||||
|
||||
void Authenticate(const std::wstring& user, const std::wstring& password);
|
||||
|
||||
|
|
|
@ -17,8 +17,10 @@
|
|||
|
||||
#include "usenetsearch/Application.h"
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
@ -26,6 +28,15 @@
|
|||
|
||||
namespace usenetsearch {
|
||||
|
||||
sig_atomic_t _signalled_to_stop{0};
|
||||
|
||||
void sigHandler(int)
|
||||
{
|
||||
_signalled_to_stop = 1;
|
||||
}
|
||||
|
||||
std::unique_ptr<Application> Application::m_instance;
|
||||
|
||||
void Application::AddBooleanOption(
|
||||
char option,
|
||||
const std::string& help,
|
||||
|
@ -88,6 +99,25 @@ void Application::AddStringOption(
|
|||
m_commandLineArguments.emplace_back(std::move(val));
|
||||
}
|
||||
|
||||
Application& Application::Get()
|
||||
{
|
||||
if (m_instance == nullptr)
|
||||
{
|
||||
m_instance = std::make_unique<Application>();
|
||||
}
|
||||
return *m_instance;
|
||||
}
|
||||
|
||||
bool Application::ShouldStop() const
|
||||
{
|
||||
const bool result = _signalled_to_stop == 1;
|
||||
if (result)
|
||||
{
|
||||
Logger::Get().Info(LOGID("Application"), "Been signalled to stop.");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void Application::Usage(const std::string& programName)
|
||||
{
|
||||
std::cout << "UsenetSearch - usenet search indexer" << std::endl;
|
||||
|
@ -147,7 +177,7 @@ void Application::Usage(const std::string& programName)
|
|||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
Application::Application() : m_db(*this), m_filter(m_config)
|
||||
Application::Application() : m_filter(m_config)
|
||||
{
|
||||
std::cout.setf(std::ios::unitbuf);
|
||||
}
|
||||
|
@ -249,6 +279,11 @@ bool Application::Init(int argc, char* argv[])
|
|||
{
|
||||
ParseArgs(argc, argv);
|
||||
if (!m_canRun) return false;
|
||||
// Install signal handlers.
|
||||
void (*prev_handler)(int);
|
||||
prev_handler = signal(SIGINT, sigHandler);
|
||||
prev_handler = signal(SIGTERM, sigHandler);
|
||||
prev_handler = signal(SIGTSTP, sigHandler);
|
||||
// Read config, setup db
|
||||
m_config.Open(m_configFile);
|
||||
m_db.MaxTreeDepth(m_config.MaxTreeDepth());
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
#include "usenetsearch/Configuration.h"
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/ScopeExit.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
|
||||
|
@ -103,7 +104,8 @@ void Configuration::Open(const std::string& filename)
|
|||
std::ifstream fin(filename.c_str());
|
||||
if (!fin.is_open())
|
||||
{
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
"Could not open configuration file: " + filename
|
||||
);
|
||||
}
|
||||
|
@ -121,7 +123,8 @@ void Configuration::Open(const std::string& filename)
|
|||
const auto kvp = StringSplit(line, std::string{":"}, 2);
|
||||
if (kvp.size() != 2)
|
||||
{
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
std::string("Invalid configuration in ")
|
||||
+ filename + std::string(" line ")
|
||||
+ std::to_string(line_nr)
|
||||
|
@ -158,7 +161,8 @@ void Configuration::Open(const std::string& filename)
|
|||
}
|
||||
catch (const std::regex_error& e)
|
||||
{
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
std::string("Invalid configuration in ")
|
||||
+ filename + std::string(" line ")
|
||||
+ std::to_string(line_nr) + " : Regular expression \""
|
||||
|
@ -175,7 +179,8 @@ void Configuration::Open(const std::string& filename)
|
|||
}
|
||||
catch (const std::regex_error& e)
|
||||
{
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
std::string("Invalid configuration in ")
|
||||
+ filename + std::string(" line ")
|
||||
+ std::to_string(line_nr) + " : Regular expression \""
|
||||
|
@ -228,7 +233,8 @@ void Configuration::Open(const std::string& filename)
|
|||
catch (const StringException& e)
|
||||
{
|
||||
fin.close();
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
std::string("Invalid configuration in ")
|
||||
+ filename + std::string(" line ")
|
||||
+ std::to_string(line_nr) + " - " + e.what()
|
||||
|
@ -241,7 +247,8 @@ void Configuration::Open(const std::string& filename)
|
|||
}
|
||||
else
|
||||
{
|
||||
throw ConfigurationException(EINVAL,
|
||||
Logger::Get().Fatal<ConfigurationException>(
|
||||
LOGID("Configuration"),
|
||||
std::string("Invalid configuration in ")
|
||||
+ filename + std::string(" line ")
|
||||
+ std::to_string(line_nr)
|
||||
|
|
|
@ -18,13 +18,13 @@
|
|||
#include "usenetsearch/Database.h"
|
||||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
#include "usenetsearch/ScopeExit.h"
|
||||
#include "usenetsearch/Serialize.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
@ -36,27 +36,42 @@ namespace usenetsearch {
|
|||
|
||||
// Database class --------------------------------------------------------------
|
||||
|
||||
Database::Database(Application& app): m_app(app)
|
||||
{
|
||||
}
|
||||
|
||||
Database::~Database()
|
||||
{
|
||||
}
|
||||
|
||||
void Database::CheckDbVersion(const SerializableFile& f) const
|
||||
{
|
||||
f.Seek(0);
|
||||
const std::uint64_t ver = f.ReadInt64();
|
||||
if (ver != m_databaseVersion)
|
||||
{
|
||||
throw DatabaseException(EINVAL,
|
||||
Logger::Get().Fatal<DatabaseException>(
|
||||
LOGID("Database"),
|
||||
"Wrong database version - Got: " + std::to_string(ver) + " want: "
|
||||
+ std::to_string(m_databaseVersion)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<NntpListEntry> Database::FindNntpEntry(std::uint64_t id)
|
||||
{
|
||||
const auto path = GetNewsGroupFilePath();
|
||||
if (!std::filesystem::exists(path)) return nullptr;
|
||||
SerializableFile io;
|
||||
io.Open(path);
|
||||
CheckDbVersion(io);
|
||||
const std::uint64_t numGroups = io.ReadInt64();
|
||||
std::unique_ptr<NntpListEntry> result = nullptr;
|
||||
for (std::uint64_t n = 0; n != numGroups; ++n)
|
||||
{
|
||||
NntpListEntry entry;
|
||||
io >> entry;
|
||||
if (entry.id == id)
|
||||
{
|
||||
result = std::make_unique<NntpListEntry>(entry);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::unique_ptr<NntpListEntry> Database::FindNntpEntry(
|
||||
const std::string& subject)
|
||||
{
|
||||
|
@ -85,8 +100,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID)
|
|||
const auto path = GetNewsGroupFilePath();
|
||||
if (!std::filesystem::exists(path))
|
||||
{
|
||||
throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: "
|
||||
+ std::to_string(newsgroupID));
|
||||
Logger::Get().Fatal<DatabaseException>(
|
||||
LOGID("Database"),
|
||||
"No indexed articles for newsgroup: "
|
||||
+ std::to_string(newsgroupID)
|
||||
);
|
||||
}
|
||||
SerializableFile io;
|
||||
io.Open(path);
|
||||
|
@ -101,8 +119,11 @@ std::uint32_t Database::GetLastIndexedArticle(std::uint64_t newsgroupID)
|
|||
return entry.lastIndexedArticle;
|
||||
}
|
||||
}
|
||||
throw DatabaseException(ENOTFOUND, "No indexed articles for newsgroup: "
|
||||
+ std::to_string(newsgroupID));
|
||||
Logger::Get().Fatal<DatabaseException>(
|
||||
LOGID("Database"),
|
||||
"No indexed articles for newsgroup: " + std::to_string(newsgroupID)
|
||||
);
|
||||
return NntpListEntry::NOT_INDEXED;
|
||||
}
|
||||
|
||||
std::filesystem::path Database::GetTokenFilePath(
|
||||
|
@ -185,8 +206,8 @@ void Database::ParseTokenFile(
|
|||
{
|
||||
if (!std::filesystem::exists(dbFile))
|
||||
{
|
||||
throw DatabaseException(
|
||||
ENOTFOUND,
|
||||
Logger::Get().Fatal<DatabaseException>(
|
||||
LOGID("Database"),
|
||||
"File does not exist: " + dbFile.string()
|
||||
);
|
||||
}
|
||||
|
@ -195,6 +216,7 @@ void Database::ParseTokenFile(
|
|||
const std::uint64_t tokenCount = io.ReadInt64();
|
||||
for (std::uint64_t i = 0; i != tokenCount; ++i)
|
||||
{
|
||||
if (Application::Get().ShouldStop()) return;
|
||||
ArticleEntry token;
|
||||
io >> token;
|
||||
onParse(token);
|
||||
|
@ -221,9 +243,11 @@ void Database::SetLastIndexedArticle(
|
|||
}
|
||||
if (!found)
|
||||
{
|
||||
throw DatabaseException(EINVAL,
|
||||
Logger::Get().Fatal<DatabaseException>(
|
||||
LOGID("Database"),
|
||||
"Attempt to update newsgroup not found in database - id: "
|
||||
+ std::to_string(newsgroupID));
|
||||
+ std::to_string(newsgroupID)
|
||||
);
|
||||
}
|
||||
UpdateNewsgroupList(*outItems);
|
||||
}
|
||||
|
@ -239,7 +263,7 @@ void Database::SaveSearchTokens(
|
|||
" ",
|
||||
m_maxTreeDepth,
|
||||
[&](const std::string& subToken, const std::string& str){
|
||||
const std::string tok = m_app.GetFilter().ProcessToken(
|
||||
const std::string tok = Application::Get().GetFilter().ProcessToken(
|
||||
subToken,
|
||||
str
|
||||
);
|
||||
|
@ -322,11 +346,6 @@ void Database::SaveToken(
|
|||
io << std::uint64_t{1};
|
||||
}
|
||||
// write out token.
|
||||
#if 0
|
||||
std::cout << "Token: " << subtoken << std::endl;
|
||||
std::cout << "Saving into file: " << path << std::endl;
|
||||
std::cout << "Token hash: " << HashBytesToString(token.hash) << std::endl << std::endl;
|
||||
#endif
|
||||
io << token;
|
||||
}
|
||||
|
||||
|
@ -352,15 +371,6 @@ std::unique_ptr<std::vector<ArticleEntry>> Database::Search(
|
|||
const auto foundTokens = LoadTokens(path, searchToken);
|
||||
if (foundTokens->empty()) continue;
|
||||
result->insert(result->end(), foundTokens->begin(), foundTokens->end());
|
||||
std::cout << std::left << std::setw(searchString.length() + 7)
|
||||
<< "token: " + searchToken
|
||||
<< std::setw(3) << " | "
|
||||
<< std::setw(10)
|
||||
<< "db file: " << path.string()
|
||||
<< std::setw(3) << " | "
|
||||
<< std::setw(9)
|
||||
<< "#results: " + std::to_string(foundTokens->size())
|
||||
<< std::endl;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
#include "usenetsearch/Dns.h"
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
|
||||
#include <cerrno>
|
||||
#include <thread>
|
||||
|
||||
|
@ -64,7 +66,8 @@ std::vector<struct addrinfo> DnsResolve(
|
|||
}
|
||||
else
|
||||
{
|
||||
throw DnsResolveException(getAddrInfoResult,
|
||||
Logger::Get().Fatal<DnsResolveException>(
|
||||
LOGID("Dns"),
|
||||
"Could not resolve host " + host + ": - Error ("
|
||||
+ std::to_string(getAddrInfoResult) + ") - "
|
||||
+ gai_strerror(getAddrInfoResult)
|
||||
|
@ -73,7 +76,8 @@ std::vector<struct addrinfo> DnsResolve(
|
|||
}
|
||||
if (result == nullptr)
|
||||
{
|
||||
throw DnsResolveException(ETIMEDOUT,
|
||||
Logger::Get().Fatal<DnsResolveException>(
|
||||
LOGID("Dns"),
|
||||
"Timed out trying to resolve host: " + host
|
||||
);
|
||||
}
|
||||
|
|
|
@ -19,17 +19,11 @@
|
|||
|
||||
namespace usenetsearch {
|
||||
|
||||
UsenetSearchException::UsenetSearchException(int errorCode, const std::string& message)
|
||||
UsenetSearchException::UsenetSearchException(const std::string& message)
|
||||
{
|
||||
m_errorCode = errorCode;
|
||||
m_message = message;
|
||||
}
|
||||
|
||||
int UsenetSearchException::Code() const
|
||||
{
|
||||
return m_errorCode;
|
||||
}
|
||||
|
||||
const char* UsenetSearchException::what() const noexcept
|
||||
{
|
||||
return m_message.c_str();
|
||||
|
|
|
@ -36,7 +36,7 @@ void Filter::Init()
|
|||
// Pre-compile regexes for all the subtokens that should be erased.
|
||||
std::for_each(eraseTokens.begin(), eraseTokens.end(),
|
||||
[&](const std::string& tok){
|
||||
const std::wstring wtok = WideStringFromString(tok);
|
||||
const std::wstring wtok = StringToLower(WideStringFromString(tok));
|
||||
m_eraseTokenRegexes.emplace(
|
||||
std::make_unique<std::wregex>(L"^" + wtok + L"\\s+"),
|
||||
std::wstring{}
|
||||
|
@ -80,17 +80,15 @@ bool Filter::ProcessNewsgroup(const std::string& newsgroup) const
|
|||
|
||||
std::string Filter::ProcessSearchString(const std::string& searchString) const
|
||||
{
|
||||
std::wstring str = WideStringFromString(searchString);
|
||||
std::wstring str = StringToLower(WideStringFromString(searchString));
|
||||
std::remove_if(str.begin(), str.end(), [](wchar_t c){
|
||||
// Remove control characters.
|
||||
if (c < 0x20) return true; // ascii control chars
|
||||
if ((c > 0x7e) && (c < 0xa0)) return true; // utf8 control chars
|
||||
return false; // don't delete anything else
|
||||
});
|
||||
// Remove Re: for obvious reasons
|
||||
str = StringRemove(StringToLower(str), std::wstring{L"re:"});
|
||||
// Remove punctuation and stuff by converting to whitespace
|
||||
static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\+\\[\\]\"\\-<>]+");
|
||||
static std::wregex rxPunctuation(L"[\\.!?#$%^&~*\\(\\)\\[\\]\"\\-<>]+");
|
||||
str = std::regex_replace(str, rxPunctuation, L" ");
|
||||
// Process erase subtoken list.
|
||||
std::for_each(m_eraseTokenRegexes.begin(), m_eraseTokenRegexes.end(),
|
||||
|
|
|
@ -17,11 +17,10 @@
|
|||
|
||||
#include "usenetsearch/Indexer.h"
|
||||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
namespace usenetsearch {
|
||||
|
||||
// SearchResult class ----------------------------------------------------------
|
||||
|
@ -110,48 +109,46 @@ bool SearchResult::operator<=(const SearchResult& other) const
|
|||
|
||||
// Indexer class ---------------------------------------------------------------
|
||||
|
||||
Indexer::Indexer(Application& app, UsenetClient& client)
|
||||
: m_app(app), m_client(client)
|
||||
Indexer::Indexer(UsenetClient& client)
|
||||
: m_client(client)
|
||||
{
|
||||
m_threads.MaxThreads(m_app.GetConfig().MaxThreads());
|
||||
m_threads.MaxThreads(Application::Get().GetConfig().MaxThreads());
|
||||
}
|
||||
|
||||
void Indexer::Connect()
|
||||
{
|
||||
m_client.Connect(
|
||||
m_app.GetConfig().NNTPServerHost(),
|
||||
m_app.GetConfig().NNTPServerPort(),
|
||||
m_app.GetConfig().NNTPServerSSL()
|
||||
Application::Get().GetConfig().NNTPServerHost(),
|
||||
Application::Get().GetConfig().NNTPServerPort(),
|
||||
Application::Get().GetConfig().NNTPServerSSL()
|
||||
);
|
||||
m_client.Authenticate(
|
||||
m_conv.from_bytes(m_app.GetConfig().NNTPServerUser()),
|
||||
m_conv.from_bytes(m_app.GetConfig().NNTPServerPassword())
|
||||
m_conv.from_bytes(Application::Get().GetConfig().NNTPServerUser()),
|
||||
m_conv.from_bytes(Application::Get().GetConfig().NNTPServerPassword())
|
||||
);
|
||||
}
|
||||
|
||||
void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
|
||||
{
|
||||
/**
|
||||
* @todo Replace all stdout stuff with Logger class.
|
||||
*/
|
||||
const size_t batchSize = m_app.GetConfig().BatchSize();
|
||||
const size_t batchSize = Application::Get().GetConfig().BatchSize();
|
||||
for (const auto& group: newsgroups)
|
||||
{
|
||||
const std::wstring newsgroup = m_conv.from_bytes(group.name);
|
||||
std::cout << "Setting group to " << group.name << "...";
|
||||
std::cout.flush();
|
||||
Logger::Get().Debug(LOGID("Indexer"), "Setting group to " + group.name);
|
||||
m_client.Group(newsgroup);
|
||||
std::cout << "DONE." << std::endl;
|
||||
std::cout << "Reading headers in " << group.name << " "
|
||||
<< "(.=" << batchSize << " headers)." << std::endl;
|
||||
std::cout.flush();
|
||||
Logger::Get().Debug(
|
||||
LOGID("Indexer"),
|
||||
"Reading headers in " + group.name + " "
|
||||
+ "(batch size = " + std::to_string(batchSize) + " headers)."
|
||||
);
|
||||
std::atomic<std::uint64_t> headerCount{0};
|
||||
const std::atomic<std::uint64_t> groupID = group.id;
|
||||
std::reference_wrapper<Database> dbref = std::ref(m_app.GetDb());
|
||||
std::uint32_t startMessage = 0;
|
||||
try
|
||||
{
|
||||
startMessage = dbref.get().GetLastIndexedArticle(groupID);
|
||||
startMessage = Application::Get().GetDb().GetLastIndexedArticle(
|
||||
groupID
|
||||
);
|
||||
if (startMessage == NntpListEntry::NOT_INDEXED)
|
||||
{
|
||||
startMessage = 0;
|
||||
|
@ -165,22 +162,27 @@ void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
|
|||
{
|
||||
startMessage = 0;
|
||||
}
|
||||
|
||||
std::cout << "Indexing starting at message: "
|
||||
<< std::to_string(startMessage) << std::endl;
|
||||
Logger::Get().Debug(
|
||||
LOGID("Indexer"),
|
||||
"Indexing starting at message: "
|
||||
+ std::to_string(startMessage)
|
||||
);
|
||||
if (Application::Get().ShouldStop()) return;
|
||||
m_client.ProcessHeaders(startMessage,
|
||||
[this, &startMessage, &headerCount, &dbref, &groupID](std::shared_ptr<NntpHeaders> headers){
|
||||
m_threads.Queue([this, headers, &startMessage, &headerCount, &dbref, &groupID](){
|
||||
[this, &startMessage, &headerCount, &groupID](std::shared_ptr<NntpHeaders> headers){
|
||||
if (Application::Get().ShouldStop()) return;
|
||||
m_threads.Queue([this, headers, &startMessage, &headerCount, &groupID](){
|
||||
std::uint64_t lastArticle{0};
|
||||
for (const auto& header: *headers)
|
||||
{
|
||||
if (Application::Get().ShouldStop()) return;
|
||||
const std::uint64_t articleID{header.articleID};
|
||||
std::string subject = header.subject;
|
||||
subject = m_app.GetFilter().ProcessSearchString(
|
||||
subject = Application::Get().GetFilter().ProcessSearchString(
|
||||
subject
|
||||
);
|
||||
if (subject == "") continue;
|
||||
dbref.get().SaveSearchTokens(
|
||||
Application::Get().GetDb().SaveSearchTokens(
|
||||
groupID,
|
||||
articleID,
|
||||
subject
|
||||
|
@ -195,20 +197,20 @@ void Indexer::Index(const std::vector<NntpListEntry>& newsgroups)
|
|||
// Update last-indexed id for the newsgroup.
|
||||
if (startMessage < lastArticle)
|
||||
{
|
||||
dbref.get().SetLastIndexedArticle(
|
||||
Application::Get().GetDb().SetLastIndexedArticle(
|
||||
groupID, lastArticle
|
||||
);
|
||||
}
|
||||
std::cout << ".";
|
||||
std::cout.flush();
|
||||
Logger::Get().Debug("Indexer::Index", "Finished batch.");
|
||||
});
|
||||
},
|
||||
batchSize
|
||||
);
|
||||
m_threads.JoinThreads();
|
||||
std::cout << "DONE." << std::endl;
|
||||
std::cout << "Saved " << headerCount << " headers." << std::endl;
|
||||
std::cout.flush();
|
||||
Logger::Get().Debug(
|
||||
LOGID("Indexer"),
|
||||
"Saved " + std::to_string(headerCount) + " headers."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -216,13 +218,21 @@ std::unique_ptr<SearchResults> Indexer::Search(
|
|||
const std::string& searchString)
|
||||
{
|
||||
auto result = std::make_unique<SearchResults>();
|
||||
const std::string sstr = m_app.GetFilter().ProcessSearchString(
|
||||
const std::string sstr = Application::Get().GetFilter().ProcessSearchString(
|
||||
searchString
|
||||
);
|
||||
auto searchResults = m_app.GetDb().Search(sstr);
|
||||
const auto searchHash = StringHashBytes(sstr);
|
||||
auto searchResults = Application::Get().GetDb().Search(sstr);
|
||||
if (!searchResults) return result;
|
||||
for(const ArticleEntry& entry: *searchResults)
|
||||
{
|
||||
if (Application::Get().ShouldStop())
|
||||
{
|
||||
Logger::Get().Fatal<UsenetSearchException>(
|
||||
"Indexer",
|
||||
"Interrupted."
|
||||
);
|
||||
}
|
||||
SearchResult sr(entry);
|
||||
// Check if a matching entry already exists in the result set, if so,
|
||||
// increment count. Otherwise, append a new entry.
|
||||
|
@ -230,6 +240,9 @@ std::unique_ptr<SearchResults> Indexer::Search(
|
|||
if (it != result->end())
|
||||
{
|
||||
(*it).Inc();
|
||||
// An exact match gets double points to ensure it's above other
|
||||
// partial matches.
|
||||
if (entry.hash == searchHash) (*it).Inc();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
#include <string>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace usenetsearch {
|
||||
|
||||
|
@ -27,7 +28,7 @@ std::unique_ptr<Logger> Logger::m_instance;
|
|||
|
||||
Logger::Logger() {}
|
||||
|
||||
Logger& Logger::get()
|
||||
Logger& Logger::Get()
|
||||
{
|
||||
if (m_instance == nullptr)
|
||||
{
|
||||
|
@ -36,19 +37,27 @@ Logger& Logger::get()
|
|||
return *m_instance;
|
||||
}
|
||||
|
||||
void Logger::debug(const std::string& msg) const
|
||||
void Logger::Debug(const std::string& func, const std::string& msg)
|
||||
{
|
||||
std::cout << "[debug] " << msg << std::endl << std::flush;
|
||||
std::lock_guard<std::mutex> lock(m_stdOutLock);
|
||||
std::cout << "[debug] [" + func + "] " << msg << std::endl << std::flush;
|
||||
}
|
||||
|
||||
void Logger::info(const std::string& msg) const
|
||||
void Logger::Info(const std::string& func, const std::string& msg)
|
||||
{
|
||||
std::cout << "[info] " << msg << std::endl << std::flush;
|
||||
std::lock_guard<std::mutex> lock(m_stdOutLock);
|
||||
std::cout << "[info] [" + func + "] " << msg << std::endl << std::flush;
|
||||
}
|
||||
|
||||
void Logger::error(const std::string& msg) const
|
||||
void Logger::Error(const std::string& func, const std::string& msg)
|
||||
{
|
||||
std::cerr << "[error] " << msg << std::endl << std::flush;
|
||||
std::lock_guard<std::mutex> lock(m_stdErrLock);
|
||||
std::cerr << "[error] [" + func + "] " << msg << std::endl << std::flush;
|
||||
}
|
||||
|
||||
std::string LogName(const std::string& module, const std::string& func)
|
||||
{
|
||||
return module + std::string{"::"} + func;
|
||||
}
|
||||
|
||||
} // namespace usenetsearch
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
#include "usenetsearch/SSLConnection.h"
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
|
||||
|
@ -38,8 +40,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret)
|
|||
if (result == SSL_ERROR_SYSCALL)
|
||||
{
|
||||
if (errno == 0) return SSLReturnState::SUCCESS;
|
||||
throw SSLException(errno, std::string{"SSL error: "}
|
||||
+ std::strerror(errno));
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
std::string{"SSL error: "} + std::strerror(errno)
|
||||
);
|
||||
}
|
||||
if (result == SSL_ERROR_SSL)
|
||||
{
|
||||
|
@ -48,8 +52,10 @@ SSLConnection::SSLReturnState SSLConnection::CheckSSLReturn(int ret)
|
|||
errorCode,
|
||||
nullptr
|
||||
);
|
||||
throw SSLException(errorCode,
|
||||
"SSL error: " + errorString);
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"SSL error: " + errorString
|
||||
);
|
||||
}
|
||||
return SSLReturnState::RETRY;
|
||||
}
|
||||
|
@ -58,14 +64,16 @@ void SSLConnection::Connect()
|
|||
{
|
||||
if (m_tcpConnection == nullptr)
|
||||
{
|
||||
throw SSLException(EBADF,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Null tcp connection when attempting ssl connect."
|
||||
);
|
||||
}
|
||||
int fd = m_tcpConnection->FileDescriptor();
|
||||
if (!fd)
|
||||
{
|
||||
throw SSLException(EBADF,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Bad file descriptor (" + std::to_string(fd)
|
||||
+ ") when attempting to ssl connect."
|
||||
);
|
||||
|
@ -93,8 +101,8 @@ void SSLConnection::Connect()
|
|||
);
|
||||
if (timeDelta > m_connectionTimeout)
|
||||
{
|
||||
throw SSLException(
|
||||
ETIMEDOUT,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Timed out while trying to establish SSL connection."
|
||||
);
|
||||
}
|
||||
|
@ -115,7 +123,8 @@ std::string SSLConnection::Read(size_t amount)
|
|||
{
|
||||
if (m_sslContext == nullptr)
|
||||
{
|
||||
throw SSLException(EBADF,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Attempted to write over SSL socket without SSL context."
|
||||
);
|
||||
}
|
||||
|
@ -155,7 +164,8 @@ void SSLConnection::Write(const std::string& data)
|
|||
{
|
||||
if (m_sslContext == nullptr)
|
||||
{
|
||||
throw SSLException(EBADF,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Attempted to write over SSL socket without SSL context."
|
||||
);
|
||||
}
|
||||
|
@ -184,8 +194,8 @@ void SSLConnection::Write(const std::string& data)
|
|||
(timeDelta > m_ioTimeout)
|
||||
|| (m_ioTimeout == std::chrono::milliseconds{0}) )
|
||||
{
|
||||
throw SSLException(
|
||||
ETIMEDOUT,
|
||||
Logger::Get().Fatal<SSLException>(
|
||||
LOGID("SSLConnection"),
|
||||
"Timed out while trying to write to SSL connection."
|
||||
);
|
||||
}
|
||||
|
|
|
@ -15,7 +15,9 @@
|
|||
|
||||
#include "usenetsearch/Serialize.h"
|
||||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Database.h"
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/ScopeExit.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
|
||||
|
@ -26,11 +28,13 @@
|
|||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <codecvt>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <locale>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace usenetsearch {
|
||||
|
@ -50,17 +54,34 @@ void SerializableFile::FileLock()
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
}
|
||||
int ret{EWOULDBLOCK};
|
||||
while ((ret == EWOULDBLOCK) || (ret == EINTR)) ret = flock(m_fd, LOCK_EX);
|
||||
while ((ret == EWOULDBLOCK) || (ret == EINTR))
|
||||
{
|
||||
if (Application::Get().ShouldStop())
|
||||
{
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Interrupt while trying to establish a lock on file ("
|
||||
+ m_fileName.string() + ")."
|
||||
);
|
||||
}
|
||||
ret = flock(m_fd, LOCK_EX);
|
||||
if ((ret == EWOULDBLOCK) || (ret == EINTR))
|
||||
{
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||
}
|
||||
}
|
||||
if (ret != 0)
|
||||
{
|
||||
throw FileIOException(
|
||||
errno, "Error trying to lock file: " + m_fileName.string()
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Error trying to lock file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
}
|
||||
|
@ -71,7 +92,8 @@ void SerializableFile::FileUnlock()
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -79,8 +101,9 @@ void SerializableFile::FileUnlock()
|
|||
const int ret = flock(m_fd, LOCK_UN);
|
||||
if (ret != 0)
|
||||
{
|
||||
throw FileIOException(
|
||||
errno, "Error trying to unlock file: " + m_fileName.string()
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Error trying to unlock file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
}
|
||||
|
@ -108,8 +131,11 @@ void SerializableFile::Open(const std::string& fileName)
|
|||
int ret = open(fileName.c_str(), flags, 0644);
|
||||
if (ret < 0)
|
||||
{
|
||||
throw FileIOException(errno, "Could not open file: " + fileName
|
||||
+ " : " + std::string{std::strerror(errno)});
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Could not open file: " + fileName
|
||||
+ " : " + std::string{std::strerror(errno)}
|
||||
);
|
||||
}
|
||||
m_fd = ret;
|
||||
if (m_lockOnOpen) FileLock();
|
||||
|
@ -119,7 +145,8 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -128,9 +155,11 @@ void SerializableFile::RangeLock(size_t offset, size_t size) const
|
|||
Seek(offset, std::ios_base::beg);
|
||||
if (lockf(m_fd, F_LOCK, size) == -1)
|
||||
{
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Could not acquire a write lock on file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno));
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
}
|
||||
Seek(pos, std::ios_base::beg);
|
||||
}
|
||||
|
@ -139,7 +168,8 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -148,9 +178,11 @@ void SerializableFile::RangeUnlock(size_t offset, size_t size) const
|
|||
Seek(offset, std::ios_base::beg);
|
||||
if (lockf(m_fd, F_ULOCK, size) == -1)
|
||||
{
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Could not release a write lock on file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno));
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
}
|
||||
Seek(pos, std::ios_base::beg);
|
||||
}
|
||||
|
@ -159,7 +191,8 @@ std::string SerializableFile::ReadStr(size_t size) const
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -176,7 +209,8 @@ std::string SerializableFile::ReadStr(size_t size) const
|
|||
const auto readNow = read(m_fd, &result[0], size);
|
||||
if (readNow == -1)
|
||||
{
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Error while reading from file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
|
@ -219,7 +253,8 @@ void SerializableFile::Seek(
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -241,7 +276,8 @@ void SerializableFile::Seek(
|
|||
const auto newOffset = lseek(m_fd, offset, whence);
|
||||
if (newOffset == -1)
|
||||
{
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Could not set cursor position in file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
|
@ -261,7 +297,8 @@ size_t SerializableFile::Tell() const
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -269,7 +306,8 @@ size_t SerializableFile::Tell() const
|
|||
const auto result = lseek(m_fd, 0, SEEK_CUR);
|
||||
if (result == -1)
|
||||
{
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Could not get cursor position in file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
|
@ -301,7 +339,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const
|
|||
{
|
||||
if (!m_fd)
|
||||
{
|
||||
throw FileIOException(EBADFD,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Attempt to write to a file (" + m_fileName.string()
|
||||
+ " ) that isn't open."
|
||||
);
|
||||
|
@ -323,7 +362,8 @@ void SerializableFile::Write(const char* bytes, size_t size) const
|
|||
{
|
||||
if (errno == 0) continue;
|
||||
}
|
||||
throw FileIOException(errno,
|
||||
Logger::Get().Fatal<FileIOException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Failure while writing to file: " + m_fileName.string()
|
||||
+ " : " + std::strerror(errno)
|
||||
);
|
||||
|
@ -452,7 +492,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj)
|
|||
in >> STX;
|
||||
if ((SOH != 1) || (STX != 2))
|
||||
{
|
||||
throw SerializeException(EBADMSG, "Bad magic number in entry header.");
|
||||
Logger::Get().Fatal<SerializeException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Bad magic number in entry header."
|
||||
);
|
||||
}
|
||||
for (std::uint8_t i = 0; i != 16; ++i)
|
||||
{
|
||||
|
@ -464,7 +507,10 @@ SerializableFile& operator>>(SerializableFile& in, ArticleEntry& obj)
|
|||
in >> EOT;
|
||||
if ((ETX != 3) || (EOT != 4))
|
||||
{
|
||||
throw SerializeException(EBADMSG, "Bad magic number in entry footer.");
|
||||
Logger::Get().Fatal<SerializeException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Bad magic number in entry footer."
|
||||
);
|
||||
}
|
||||
return in;
|
||||
}
|
||||
|
@ -509,7 +555,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj)
|
|||
in >> STX;
|
||||
if ((SOH != 1) || (STX != 2))
|
||||
{
|
||||
throw SerializeException(EBADMSG,
|
||||
Logger::Get().Fatal<SerializeException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Bad magic number in NNTP entry header."
|
||||
);
|
||||
}
|
||||
|
@ -524,7 +571,8 @@ SerializableFile& operator>>(SerializableFile& in, NntpListEntry& obj)
|
|||
in >> EOT;
|
||||
if ((ETX != 3) || (EOT != 4))
|
||||
{
|
||||
throw SerializeException(EBADMSG,
|
||||
Logger::Get().Fatal<SerializeException>(
|
||||
LOGID("SerializableFile"),
|
||||
"Bad magic number in NNTP entry footer."
|
||||
);
|
||||
}
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
|
||||
#include <openssl/md5.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
@ -111,9 +113,11 @@ bool StringToBoolean(const std::string& str)
|
|||
const std::string lstr = StringTrim(StringToLower(str));
|
||||
if ((lstr == "true") || (lstr == "yes") || (lstr == "1")) return true;
|
||||
if ((lstr == "false") || (lstr == "no") || (lstr == "0")) return false;
|
||||
throw StringException(EINVAL,
|
||||
Logger::Get().Fatal<StringException>(
|
||||
LOGID("StringUtils"),
|
||||
"The string \"" + str + "\" is not a valid boolean value."
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool StringToBoolean(const std::wstring& str)
|
||||
|
@ -122,10 +126,12 @@ bool StringToBoolean(const std::wstring& str)
|
|||
if ((lstr == L"true") || (lstr == L"yes") || (lstr == L"1")) return true;
|
||||
if ((lstr == L"false") || (lstr == L"no") || (lstr == L"0")) return false;
|
||||
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv;
|
||||
throw StringException(EINVAL,
|
||||
Logger::Get().Fatal<StringException>(
|
||||
LOGID("StringUtils"),
|
||||
"The string \"" + conv.to_bytes(str)
|
||||
+ "\" is not a valid boolean value."
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
void StringTreeOperation(
|
||||
|
@ -135,6 +141,7 @@ void StringTreeOperation(
|
|||
std::function<void(const std::string& subToken, const std::string& str)> Fn)
|
||||
{
|
||||
const auto tokens = StringSplit(searchString, splitBy);
|
||||
std::vector<std::string> tokenList;
|
||||
for (auto outerIt = tokens.begin(); outerIt != tokens.end(); outerIt++)
|
||||
{
|
||||
for (size_t depth = 1; depth != maxDepth + 1; ++depth)
|
||||
|
@ -142,7 +149,16 @@ void StringTreeOperation(
|
|||
const auto endIt = outerIt + depth;
|
||||
const auto subset = std::vector<std::string>(outerIt, endIt);
|
||||
const auto subToken = StringJoin(subset, splitBy);
|
||||
// Check if we already have this token.
|
||||
//
|
||||
// For phrases like "we went here and then we went there" this would
|
||||
// avoid indexing the tokens 'we' and 'went' and 'we went' twice.
|
||||
if (std::find(tokenList.begin(), tokenList.end(), subToken)
|
||||
== tokenList.end())
|
||||
{
|
||||
Fn(subToken, searchString);
|
||||
tokenList.emplace_back(subToken);
|
||||
}
|
||||
if (endIt == tokens.end()) break;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
along with UsenetSearch. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/TcpConnection.h"
|
||||
|
||||
#include "usenetsearch/Dns.h"
|
||||
|
@ -45,7 +46,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
|
|||
const std::vector<struct addrinfo> addresses = DnsResolve(host, port);
|
||||
if (addresses.empty())
|
||||
{
|
||||
throw DnsResolveException(EDESTADDRREQ,
|
||||
Logger::Get().Fatal<DnsResolveException>(
|
||||
LOGID("TcpConnection"),
|
||||
"The provided host (" + host + ") did not resolve to an address."
|
||||
);
|
||||
}
|
||||
|
@ -66,8 +68,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
|
|||
);
|
||||
if (timeDelta > m_connectionTimeout)
|
||||
{
|
||||
throw SocketException(
|
||||
ETIMEDOUT,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Timed out while trying to connect to " + host + ":"
|
||||
+ std::to_string(port) + "."
|
||||
);
|
||||
|
@ -75,7 +77,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
|
|||
fd = socket(addr.ai_family, SOCK_STREAM, 0);
|
||||
if (fd < 0)
|
||||
{
|
||||
throw SocketException(errno,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Failed to create socket - Error (" + std::to_string(errno)
|
||||
+ ") - " + std::strerror(errno)
|
||||
);
|
||||
|
@ -101,7 +104,8 @@ void TcpConnection::Connect(const std::string& host, std::uint16_t port)
|
|||
else
|
||||
{
|
||||
close(fd);
|
||||
throw SocketException(errno,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Failed to connect to " + host + ":"
|
||||
+ std::to_string(port) + " - Error ("
|
||||
+ std::to_string(errno) + ") - " + strerror(errno)
|
||||
|
@ -152,7 +156,8 @@ std::string TcpConnection::Read(size_t amount)
|
|||
}
|
||||
else
|
||||
{
|
||||
throw SocketException(errno,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Error while reading from TCP socket (" + std::to_string(errno)
|
||||
+ ") - " + std::strerror(errno)
|
||||
);
|
||||
|
@ -178,7 +183,8 @@ void TcpConnection::Write(const std::string& data)
|
|||
if ((timeDelta > m_ioTimeout)
|
||||
|| m_ioTimeout == std::chrono::milliseconds{0})
|
||||
{
|
||||
throw SocketException(ETIMEDOUT,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Timed out writing to TCP socket."
|
||||
);
|
||||
}
|
||||
|
@ -191,7 +197,8 @@ void TcpConnection::Write(const std::string& data)
|
|||
}
|
||||
else
|
||||
{
|
||||
throw SocketException(errno,
|
||||
Logger::Get().Fatal<SocketException>(
|
||||
LOGID("TcpConnection"),
|
||||
"Error writing to tcp socket (" + std::to_string(errno)
|
||||
+ ") - " + std::strerror(errno)
|
||||
);
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Except.h"
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
|
||||
#include <codecvt>
|
||||
|
@ -32,10 +33,6 @@ namespace usenetsearch {
|
|||
|
||||
// UsenetClient class ----------------------------------------------------------
|
||||
|
||||
UsenetClient::UsenetClient(Application& app): m_app(app)
|
||||
{
|
||||
}
|
||||
|
||||
void UsenetClient::Authenticate(
|
||||
const std::wstring& user,
|
||||
const std::wstring& password)
|
||||
|
@ -45,8 +42,8 @@ void UsenetClient::Authenticate(
|
|||
auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error authenticating with NNTP server: "
|
||||
+ response.message
|
||||
);
|
||||
|
@ -56,8 +53,8 @@ void UsenetClient::Authenticate(
|
|||
response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error authenticating with NNTP server: "
|
||||
+ response.message
|
||||
);
|
||||
|
@ -83,8 +80,8 @@ void UsenetClient::Connect(
|
|||
}
|
||||
catch (const UsenetSearchException& e)
|
||||
{
|
||||
throw UsenetClientException(
|
||||
e.Code(),
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error while trying to connect to host: " + host + ":"
|
||||
+ std::to_string(port) + " - " + e.what()
|
||||
);
|
||||
|
@ -93,8 +90,8 @@ void UsenetClient::Connect(
|
|||
const auto serverHello = ReadLine();
|
||||
if (IsError(serverHello))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
serverHello.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error received from NNTP server: "
|
||||
+ serverHello.message
|
||||
);
|
||||
|
@ -108,8 +105,8 @@ void UsenetClient::Group(const std::wstring& newsgroup)
|
|||
auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error changing group to " + StringFromWideString(newsgroup) + " : "
|
||||
+ response.message
|
||||
);
|
||||
|
@ -124,8 +121,8 @@ NntpHeader UsenetClient::Head(std::uint64_t articleID)
|
|||
auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error getting headers for article id "
|
||||
+ std::to_string(articleID)
|
||||
+ " : " + response.message
|
||||
|
@ -164,8 +161,8 @@ std::unique_ptr<std::vector<NntpListEntry>> UsenetClient::List()
|
|||
const auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Failed to fetch newsgroup list from server, "
|
||||
+ std::string{"server responded with: "}
|
||||
+ response.message
|
||||
|
@ -189,7 +186,7 @@ std::unique_ptr<std::vector<NntpListEntry>> UsenetClient::List()
|
|||
entry.status = fields[4];
|
||||
entry.id = 0; // incremented by db when saving.
|
||||
entry.lastIndexedArticle = NntpListEntry::NOT_INDEXED;
|
||||
if (m_app.GetFilter().ProcessNewsgroup(entry.name))
|
||||
if (Application::Get().GetFilter().ProcessNewsgroup(entry.name))
|
||||
{
|
||||
result->emplace_back(entry);
|
||||
}
|
||||
|
@ -202,7 +199,9 @@ std::unique_ptr<std::vector<std::uint64_t>> UsenetClient::ListGroup(
|
|||
const std::wstring& newsGroup)
|
||||
{
|
||||
auto result = std::make_unique<std::vector<std::uint64_t>>();
|
||||
if (!m_app.GetFilter().ProcessNewsgroup(StringFromWideString(newsGroup)))
|
||||
if (!Application::Get().GetFilter().ProcessNewsgroup(
|
||||
StringFromWideString(newsGroup))
|
||||
)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
@ -212,8 +211,8 @@ std::unique_ptr<std::vector<std::uint64_t>> UsenetClient::ListGroup(
|
|||
const auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Failed to fetch newsgroup list from server, "
|
||||
+ std::string{"server responded with: "}
|
||||
+ response.message
|
||||
|
@ -241,8 +240,8 @@ void UsenetClient::ProcessHeaders(
|
|||
const auto response = ReadLine();
|
||||
if (IsError(response))
|
||||
{
|
||||
throw UsenetClientException(
|
||||
response.code,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"Error getting headers: "
|
||||
+ response.message
|
||||
);
|
||||
|
@ -293,9 +292,11 @@ NntpMessage UsenetClient::ReadLine()
|
|||
line = ReadUntil("\r\n");
|
||||
if (line.length() < 2)
|
||||
{
|
||||
throw UsenetSearchException(EPROTONOSUPPORT,
|
||||
Logger::Get().Fatal<UsenetClientException>(
|
||||
LOGID("UsenetClient"),
|
||||
"NNTP protocol error - invalid response from server: "
|
||||
+ line);
|
||||
+ line
|
||||
);
|
||||
}
|
||||
std::string codeStr = line.substr(0, 3);
|
||||
result.code = std::stoi(codeStr);
|
||||
|
|
|
@ -25,10 +25,9 @@ using namespace usenetsearch;
|
|||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
Application app;
|
||||
std::string tokenFile{""};
|
||||
std::string newsgroupFile{""};
|
||||
app.AddFileOption(
|
||||
Application::Get().AddFileOption(
|
||||
't',
|
||||
"token db file to dump.",
|
||||
[&tokenFile](const std::string& val)
|
||||
|
@ -36,7 +35,7 @@ int main(int argc, char* argv[])
|
|||
tokenFile = val;
|
||||
}
|
||||
);
|
||||
app.AddFileOption(
|
||||
Application::Get().AddFileOption(
|
||||
'n',
|
||||
"newsgroup file to dump.",
|
||||
[&newsgroupFile](const std::string& val)
|
||||
|
@ -44,20 +43,24 @@ int main(int argc, char* argv[])
|
|||
newsgroupFile = val;
|
||||
}
|
||||
);
|
||||
if (!app.Init(argc, argv)) return 1;
|
||||
if (!Application::Get().Init(argc, argv)) return 1;
|
||||
if (!tokenFile.empty())
|
||||
{
|
||||
app.GetDb().ParseTokenFile(tokenFile, [](const ArticleEntry& token){
|
||||
Application::Get().GetDb().ParseTokenFile(
|
||||
tokenFile,
|
||||
[](const ArticleEntry& token){
|
||||
std::cout << "Hash: " << HashBytesToString(token.hash) << " | "
|
||||
<< "NewsgroupID: " << token.newsgroupID << " | "
|
||||
<< "ArticleID: " << token.articleID << std::endl;
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
if (!newsgroupFile.empty())
|
||||
{
|
||||
const auto groups = app.GetDb().LoadNewsgroupList();
|
||||
const auto groups = Application::Get().GetDb().LoadNewsgroupList();
|
||||
for(const auto& group: *groups)
|
||||
{
|
||||
if (Application::Get().ShouldStop()) return 1;
|
||||
std::cout << std::left
|
||||
<< std::setw(9) << "Id: " + std::to_string(group.id)
|
||||
<< std::setw(3) << " | "
|
||||
|
|
|
@ -1,7 +1,8 @@
|
|||
|
||||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
#include "usenetsearch/Indexer.h"
|
||||
#include "usenetsearch/StringUtils.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
|
@ -9,28 +10,27 @@ using namespace usenetsearch;
|
|||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
Application app;
|
||||
std::string searchString;
|
||||
app.AddStringOption('s', "Search string",
|
||||
Application::Get().AddStringOption('s', "Search string",
|
||||
[&searchString](const std::string& s){
|
||||
searchString = s;
|
||||
}
|
||||
);
|
||||
int maxResults{0};
|
||||
app.AddIntegerOption('n', "Maximum results",
|
||||
Application::Get().AddIntegerOption('n', "Maximum results",
|
||||
[&maxResults](int n){
|
||||
maxResults = n;
|
||||
}
|
||||
);
|
||||
if (!app.Init(argc, argv)) return 1;
|
||||
if (!Application::Get().Init(argc, argv)) return 1;
|
||||
if (searchString.empty())
|
||||
{
|
||||
std::cerr << "Missing search string." << std::endl;
|
||||
app.Usage(argv[0]);
|
||||
Application::Get().Usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
UsenetClient client(app);
|
||||
Indexer idx(app, client);
|
||||
UsenetClient client;
|
||||
Indexer idx(client);
|
||||
std::unique_ptr<SearchResults> results = idx.Search(
|
||||
searchString
|
||||
);
|
||||
|
@ -39,15 +39,24 @@ int main(int argc, char* argv[])
|
|||
std::cout << "Nothing found." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
idx.Connect();
|
||||
size_t resultCounter{0};
|
||||
for (const auto& sr: *results)
|
||||
{
|
||||
std::cout << std::left
|
||||
<< std::setw(18) << "Newsgroup id: " + std::to_string(sr.NewsgroupId())
|
||||
<< std::setw(4) << " | "
|
||||
<< std::setw(17) << "Article id: " + std::to_string(sr.ArticleId())
|
||||
<< std::setw(4) << " | "
|
||||
<< std::setw(10) << "Hits: " + std::to_string(sr.Hits())
|
||||
if (Application::Get().ShouldStop()) return 1;
|
||||
const std::unique_ptr<NntpListEntry> newsGroup
|
||||
= Application::Get().GetDb().FindNntpEntry(sr.NewsgroupId());
|
||||
if (!newsGroup) continue;
|
||||
client.Group(WideStringFromString(newsGroup->name));
|
||||
const NntpHeader headers = client.Head(sr.ArticleId());
|
||||
std::cout
|
||||
<< "Score: " + std::to_string(sr.Hits())
|
||||
<< " | "
|
||||
<< "Newsgroup: " + newsGroup->name
|
||||
<< " | "
|
||||
<< "Article id: " + std::to_string(sr.ArticleId())
|
||||
<< " | "
|
||||
<< "Subject: " + headers.subject
|
||||
<< std::endl;
|
||||
resultCounter++;
|
||||
if ((maxResults > 0) && (resultCounter >= maxResults)) break;
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
#include "usenetsearch/Application.h"
|
||||
#include "usenetsearch/Except.h"
|
||||
#include "usenetsearch/Indexer.h"
|
||||
#include "usenetsearch/Logger.h"
|
||||
#include "usenetsearch/UsenetClient.h"
|
||||
|
||||
#include <iostream>
|
||||
|
@ -26,30 +27,30 @@ using namespace usenetsearch;
|
|||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
Application app;
|
||||
if (!app.Init(argc, argv)) return 1;
|
||||
if (!Application::Get().Init(argc, argv)) return 1;
|
||||
|
||||
UsenetClient client(app);
|
||||
Indexer indexer(app, client);
|
||||
std::cout << "Connecting to newsgroup server...";
|
||||
UsenetClient client;
|
||||
Indexer indexer(client);
|
||||
Logger::Get().Debug(
|
||||
LOGID("usenetindexd"),
|
||||
"Connecting to newsgroup server..."
|
||||
);
|
||||
indexer.Connect();
|
||||
std::cout << "<OK>" << std::endl;
|
||||
Logger::Get().Debug(LOGID("usenetindexd"), "Connected.");
|
||||
|
||||
try
|
||||
{
|
||||
std::cout << "Getting newsgroup list...";
|
||||
std::cout.flush();
|
||||
Logger::Get().Debug(LOGID("usenetindexd"), "Getting newsgroup list...");
|
||||
auto list = client.List();
|
||||
app.GetDb().UpdateNewsgroupList(*list);
|
||||
std::cout << "<DONE>" << std::endl;
|
||||
std::cout.flush();
|
||||
std::cout << "Found " << list->size() << " newsgroups." << std::endl;
|
||||
std::cout.flush();
|
||||
Application::Get().GetDb().UpdateNewsgroupList(*list);
|
||||
Logger::Get().Debug(
|
||||
LOGID("usenetindexd"),
|
||||
"Found " + std::to_string(list->size()) + " newsgroups."
|
||||
);
|
||||
indexer.Index(*list);
|
||||
}
|
||||
catch (const UsenetSearchException& e)
|
||||
{
|
||||
std::cerr << e.what() << std::endl;;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
|
|
Loading…
Reference in New Issue