UsenetSearch/src/usenetindexd.cpp

/*
    Copyright© 2021 John Sennesael

    UsenetSearch is Free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    UsenetSearch is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with UsenetSearch.  If not, see <https://www.gnu.org/licenses/>.
*/

#include <iostream>

#include "usenetsearch/Application.h"
#include "usenetsearch/Except.h"
#include "usenetsearch/Indexer.h"
#include "usenetsearch/UsenetClient.h"

using namespace usenetsearch;

int main(int argc, char* argv[])
{
    Application app;
    app.Init(argc, argv);

    UsenetClient client;

    Indexer indexer(app, client);
    indexer.Connect();


    try 
    {
        // BEGIN TEMPORARY TEST CODE
        std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv;
        std::unique_ptr<std::vector<NntpListEntry>> list;
        NntpListEntry e{};
        e.count = 100;

// 1001 headers
//        e.name = "comp.os.os2.comm";

// 2541 headers
//        e.name = "borland.public.cppbuilder.commandlinetools";

// 100026 headers (1859952 K) (1816.35 M)
//        e.name = "dk.videnskab";
// 1000437 headers
        e.name = "alt.bible";

// a million or so, but this one is very slow because all subjects look the
// same, so everything goes to the same token index, which means we're 
// constantly waiting on a file lock.
//        e.name = "usenetserver.test";

        list = std::make_unique<std::vector<NntpListEntry>>();
        list->emplace_back(e);
        if ((list == nullptr) || (list->empty()))
        {
            std::cout << "Getting newsgroup list...";
            std::cout.flush();
            list = client.List();
            app.GetDb().UpdateNewsgroupList(*list);
            std::cout << "DONE." << std::endl;
            std::cout.flush();
        }
        std::cout << "Number of newsgroups in newsgroup: "
                  << list->size() << std::endl;
        std::cout.flush();
        // END TEMPORARY TEST CODE
        indexer.Index(*list);
    }
    catch (const UsenetSearchException& e)
    {
        std::cerr << e.what() << std::endl;;
        return 1;
    }
    return 0;
}