2021-09-21 01:02:23 +00:00
|
|
|
/*
|
2021-09-21 22:46:31 +00:00
|
|
|
Copyright© 2021 John Sennesael
|
|
|
|
|
2021-10-08 20:17:22 +00:00
|
|
|
UsenetSearch is Free software: you can redistribute it and/or modify
|
2021-09-21 01:02:23 +00:00
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
UsenetSearch is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with UsenetSearch. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2021-10-20 12:18:20 +00:00
|
|
|
#include "usenetsearch/UsenetClient.h"
|
|
|
|
|
|
|
|
#include "usenetsearch/Application.h"
|
|
|
|
#include "usenetsearch/Except.h"
|
2021-10-20 22:48:43 +00:00
|
|
|
#include "usenetsearch/Logger.h"
|
2021-10-20 12:18:20 +00:00
|
|
|
#include "usenetsearch/StringUtils.h"
|
|
|
|
|
2021-09-19 21:05:16 +00:00
|
|
|
#include <codecvt>
|
2021-09-21 00:48:49 +00:00
|
|
|
#include <fstream>
|
2021-09-19 21:05:16 +00:00
|
|
|
#include <locale>
|
|
|
|
#include <memory>
|
2021-09-29 23:52:54 +00:00
|
|
|
#include <mutex>
|
2021-09-19 21:05:16 +00:00
|
|
|
#include <string>
|
|
|
|
|
|
|
|
namespace usenetsearch {
|
|
|
|
|
2021-09-21 00:48:49 +00:00
|
|
|
// UsenetClient class ----------------------------------------------------------
|
|
|
|
|
2021-09-19 21:05:16 +00:00
|
|
|
void UsenetClient::Authenticate(
|
|
|
|
const std::wstring& user,
|
|
|
|
const std::wstring& password)
|
|
|
|
{
|
|
|
|
// Send user name
|
|
|
|
Write(L"AUTHINFO USER " + user + L"\r\n");
|
|
|
|
auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-19 21:05:16 +00:00
|
|
|
"Error authenticating with NNTP server: "
|
2021-09-29 23:52:54 +00:00
|
|
|
+ response.message
|
2021-09-19 21:05:16 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
// Send password
|
|
|
|
Write(L"AUTHINFO PASS " + password + L"\r\n");
|
|
|
|
response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-19 21:05:16 +00:00
|
|
|
"Error authenticating with NNTP server: "
|
2021-09-29 23:52:54 +00:00
|
|
|
+ response.message
|
2021-09-19 21:05:16 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void UsenetClient::Connect(
|
|
|
|
const std::string& host,
|
|
|
|
std::uint16_t port,
|
|
|
|
bool useSSL)
|
|
|
|
{
|
|
|
|
// Establish connection.
|
|
|
|
m_useSSL = useSSL;
|
|
|
|
try
|
|
|
|
{
|
|
|
|
m_tcp = std::make_unique<usenetsearch::TcpConnection>();
|
|
|
|
m_tcp->Connect(host, port);
|
|
|
|
if (useSSL)
|
|
|
|
{
|
|
|
|
m_ssl = std::make_unique<SSLConnection>(std::move(m_tcp));
|
|
|
|
m_ssl->Connect();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
catch (const UsenetSearchException& e)
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-19 21:05:16 +00:00
|
|
|
"Error while trying to connect to host: " + host + ":"
|
|
|
|
+ std::to_string(port) + " - " + e.what()
|
|
|
|
);
|
|
|
|
}
|
|
|
|
// Read server banner.
|
|
|
|
const auto serverHello = ReadLine();
|
|
|
|
if (IsError(serverHello))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-19 21:05:16 +00:00
|
|
|
"Error received from NNTP server: "
|
2021-09-29 23:52:54 +00:00
|
|
|
+ serverHello.message
|
2021-09-19 21:05:16 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-29 23:52:54 +00:00
|
|
|
void UsenetClient::Group(const std::wstring& newsgroup)
|
2021-09-21 22:46:31 +00:00
|
|
|
{
|
|
|
|
// Send user name
|
2021-09-29 23:52:54 +00:00
|
|
|
Write(L"GROUP " + newsgroup + L"\r\n");
|
2021-09-21 22:46:31 +00:00
|
|
|
auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-10-19 01:19:11 +00:00
|
|
|
"Error changing group to " + StringFromWideString(newsgroup) + " : "
|
2021-09-29 23:52:54 +00:00
|
|
|
+ response.message
|
2021-09-21 22:46:31 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-29 23:52:54 +00:00
|
|
|
NntpHeader UsenetClient::Head(std::uint64_t articleID)
|
2021-09-22 00:19:13 +00:00
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
Write(L"HEAD " + std::to_wstring(articleID) + L"\r\n");
|
2021-09-22 00:19:13 +00:00
|
|
|
/* Typical response is a code 221 followed by the headers ending with a
|
|
|
|
line containing a period by itself. */
|
|
|
|
auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-22 00:19:13 +00:00
|
|
|
"Error getting headers for article id "
|
2021-09-29 23:52:54 +00:00
|
|
|
+ std::to_string(articleID)
|
|
|
|
+ " : " + response.message
|
2021-09-22 00:19:13 +00:00
|
|
|
);
|
|
|
|
}
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto headerLinesStr = ReadUntil("\r\n.\r\n");
|
2021-09-22 00:19:13 +00:00
|
|
|
// parse the headers.
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto lines = StringSplit(headerLinesStr, std::string{"\r\n"});
|
2021-09-22 00:19:13 +00:00
|
|
|
NntpHeader result;
|
|
|
|
result.articleID = articleID;
|
|
|
|
for (const auto& line: lines)
|
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto kvp = StringSplit(line, std::string{":"}, 2);
|
2021-09-22 00:19:13 +00:00
|
|
|
if (kvp.size() != 2) continue; // Bad header line?
|
|
|
|
const auto key = StringToLower(StringTrim(kvp[0]));
|
|
|
|
const auto value = StringTrim(kvp[1]);
|
2021-09-29 23:52:54 +00:00
|
|
|
if (key == "subject")
|
2021-09-22 00:19:13 +00:00
|
|
|
{
|
|
|
|
result.subject = value;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-09-19 21:05:16 +00:00
|
|
|
bool UsenetClient::IsError(const NntpMessage& msg) const
|
|
|
|
{
|
|
|
|
if (msg.code >= 400) return true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2021-09-21 22:46:31 +00:00
|
|
|
std::unique_ptr<std::vector<NntpListEntry>> UsenetClient::List()
|
2021-09-21 00:48:49 +00:00
|
|
|
{
|
|
|
|
Write(L"LIST COUNTS\r\n");
|
|
|
|
/* In response, we should get a 215 response followed by the list of news
|
|
|
|
groups ending in a period on it's own line. */
|
|
|
|
const auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-21 00:48:49 +00:00
|
|
|
"Failed to fetch newsgroup list from server, "
|
|
|
|
+ std::string{"server responded with: "}
|
2021-09-29 23:52:54 +00:00
|
|
|
+ response.message
|
2021-09-21 00:48:49 +00:00
|
|
|
);
|
|
|
|
}
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto listStr = ReadUntil("\r\n.\r\n");
|
2021-09-21 00:48:49 +00:00
|
|
|
// parse the list.
|
2021-09-29 23:52:54 +00:00
|
|
|
auto lines = StringSplit(listStr, std::string{"\r\n"});
|
2021-09-21 22:46:31 +00:00
|
|
|
auto result = std::make_unique<std::vector<NntpListEntry>>();
|
2021-09-22 00:19:13 +00:00
|
|
|
if (lines.empty()) return result;
|
2021-09-21 00:48:49 +00:00
|
|
|
for (const auto& line: lines)
|
|
|
|
{
|
|
|
|
NntpListEntry entry;
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto fields = StringSplit(line, std::string{" "});
|
2021-09-21 00:48:49 +00:00
|
|
|
if (fields.size() == 5)
|
|
|
|
{
|
|
|
|
entry.name = fields[0];
|
|
|
|
entry.high = std::stoul(fields[1]);
|
|
|
|
entry.low = std::stoul(fields[2]);
|
|
|
|
entry.count = std::stoul(fields[3]);
|
|
|
|
entry.status = fields[4];
|
2021-10-19 01:19:11 +00:00
|
|
|
entry.id = 0; // incremented by db when saving.
|
2021-10-19 22:56:28 +00:00
|
|
|
entry.lastIndexedArticle = NntpListEntry::NOT_INDEXED;
|
2021-10-20 22:48:43 +00:00
|
|
|
if (Application::Get().GetFilter().ProcessNewsgroup(entry.name))
|
2021-10-19 01:19:11 +00:00
|
|
|
{
|
|
|
|
result->emplace_back(entry);
|
|
|
|
}
|
2021-09-21 00:48:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-09-29 23:52:54 +00:00
|
|
|
std::unique_ptr<std::vector<std::uint64_t>> UsenetClient::ListGroup(
|
|
|
|
const std::wstring& newsGroup)
|
2021-09-21 22:46:31 +00:00
|
|
|
{
|
2021-10-19 01:19:11 +00:00
|
|
|
auto result = std::make_unique<std::vector<std::uint64_t>>();
|
2021-10-20 22:48:43 +00:00
|
|
|
if (!Application::Get().GetFilter().ProcessNewsgroup(
|
|
|
|
StringFromWideString(newsGroup))
|
|
|
|
)
|
2021-10-19 01:19:11 +00:00
|
|
|
{
|
|
|
|
return result;
|
|
|
|
}
|
2021-09-21 22:46:31 +00:00
|
|
|
Write(L"LISTGROUP " + newsGroup + L"\r\n");
|
|
|
|
/* In response, we should get a 211 response followed by the list of
|
|
|
|
article ID's ending in a period on it's own line. */
|
|
|
|
const auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-21 22:46:31 +00:00
|
|
|
"Failed to fetch newsgroup list from server, "
|
|
|
|
+ std::string{"server responded with: "}
|
2021-09-29 23:52:54 +00:00
|
|
|
+ response.message
|
2021-09-21 22:46:31 +00:00
|
|
|
);
|
|
|
|
}
|
2021-09-29 23:52:54 +00:00
|
|
|
const auto listStr = ReadUntil("\r\n.\r\n");
|
2021-09-21 22:46:31 +00:00
|
|
|
// parse the list.
|
2021-09-29 23:52:54 +00:00
|
|
|
auto lines = StringSplit(listStr, std::string{"\r\n"});
|
2021-09-22 00:19:13 +00:00
|
|
|
if (lines.empty()) return result;
|
2021-09-21 22:46:31 +00:00
|
|
|
for (const auto& line: lines)
|
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
result->emplace_back(stoul(StringTrim(line)));
|
2021-09-21 22:46:31 +00:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-10-19 01:19:11 +00:00
|
|
|
void UsenetClient::ProcessHeaders(
|
|
|
|
std::uint64_t startMessage,
|
|
|
|
std::function<void(std::shared_ptr<NntpHeaders>)> processFn,
|
|
|
|
std::uint64_t batchSize)
|
|
|
|
{
|
|
|
|
Write(L"XHDR subject " + std::to_wstring(startMessage) + L"-\r\n");
|
|
|
|
/* Typical response is a code 221 followed by the headers ending with a
|
|
|
|
line containing a period by itself. */
|
|
|
|
const auto response = ReadLine();
|
|
|
|
if (IsError(response))
|
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-10-19 01:19:11 +00:00
|
|
|
"Error getting headers: "
|
|
|
|
+ response.message
|
|
|
|
);
|
|
|
|
}
|
|
|
|
NntpHeaders headers;
|
|
|
|
std::mutex headersLock;
|
|
|
|
while (true)
|
|
|
|
{
|
|
|
|
const auto line = StringTrim(ReadUntil("\r\n"));
|
|
|
|
if (line == ".") break;
|
|
|
|
// parse the headers.
|
|
|
|
const auto kvp = StringSplit(line, std::string{" "}, 2);
|
|
|
|
if (kvp.size() != 2) continue; // Bad header line?
|
|
|
|
const std::uint64_t articleID = std::stoul(StringTrim(kvp[0]));
|
|
|
|
const auto subject = StringTrim(kvp[1]);
|
|
|
|
NntpHeader hdr;
|
|
|
|
hdr.articleID = articleID;
|
|
|
|
hdr.subject = subject;
|
|
|
|
{
|
|
|
|
std::lock_guard<std::mutex> lock(headersLock);
|
|
|
|
headers.emplace_back(hdr);
|
|
|
|
}
|
|
|
|
if (headers.size() >= batchSize)
|
|
|
|
{
|
|
|
|
auto headersToPass = std::make_shared<NntpHeaders>(
|
|
|
|
headers.begin(),
|
|
|
|
headers.end()
|
|
|
|
);
|
|
|
|
processFn(std::move(headersToPass));
|
|
|
|
headers.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Left over headers.
|
|
|
|
if (!headers.empty())
|
|
|
|
{
|
|
|
|
auto headersToPass = std::make_shared<NntpHeaders>(
|
|
|
|
headers.begin(),
|
|
|
|
headers.end()
|
|
|
|
);
|
|
|
|
processFn(std::move(headersToPass));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-19 21:05:16 +00:00
|
|
|
NntpMessage UsenetClient::ReadLine()
|
|
|
|
{
|
|
|
|
NntpMessage result{};
|
2021-09-29 23:52:54 +00:00
|
|
|
std::string line;
|
|
|
|
line = ReadUntil("\r\n");
|
|
|
|
if (line.length() < 2)
|
2021-09-19 21:05:16 +00:00
|
|
|
{
|
2021-10-20 22:48:43 +00:00
|
|
|
Logger::Get().Fatal<UsenetClientException>(
|
|
|
|
LOGID("UsenetClient"),
|
2021-09-19 21:05:16 +00:00
|
|
|
"NNTP protocol error - invalid response from server: "
|
2021-10-20 22:48:43 +00:00
|
|
|
+ line
|
|
|
|
);
|
2021-09-19 21:05:16 +00:00
|
|
|
}
|
2021-09-29 23:52:54 +00:00
|
|
|
std::string codeStr = line.substr(0, 3);
|
2021-09-19 21:05:16 +00:00
|
|
|
result.code = std::stoi(codeStr);
|
|
|
|
result.message = line.substr(4, line.length());
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-09-29 23:52:54 +00:00
|
|
|
std::string UsenetClient::ReadUntil(const std::string& deliminator)
|
2021-09-19 21:05:16 +00:00
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
std::string result;
|
2021-09-19 21:05:16 +00:00
|
|
|
if (m_useSSL)
|
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
result = m_ssl->ReadUntil(deliminator);
|
2021-09-19 21:05:16 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-09-29 23:52:54 +00:00
|
|
|
result = m_tcp->ReadUntil(deliminator);
|
2021-09-22 00:19:13 +00:00
|
|
|
}
|
2021-09-29 23:52:54 +00:00
|
|
|
result.erase(result.size() - deliminator.size(), deliminator.size());
|
2021-09-19 21:05:16 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
void UsenetClient::Write(const std::wstring& message)
|
|
|
|
{
|
2021-10-19 01:19:11 +00:00
|
|
|
const std::string toSend = StringFromWideString(message);
|
2021-09-19 21:05:16 +00:00
|
|
|
if (m_useSSL)
|
|
|
|
{
|
|
|
|
m_ssl->Write(toSend);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
m_tcp->Write(toSend);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace usenetsearch
|