diff --git a/.gitignore b/.gitignore index cbe70c5..ef69d74 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ node_modules/ package-lock.json skor +*.swp +*.swo +*.swn diff --git a/readme.md b/readme.md index 44c7a97..e057d41 100644 --- a/readme.md +++ b/readme.md @@ -3,25 +3,53 @@ a script to get live scores from hltv.org -## why? -- its really hard to scrape hltv live scores (i tried) - ## requirements +- node.js +- npm +- `http` node package - nightmare - electron - - if npm is giving you perm errors try `sudo npm install -g electron --unsafe-perm=true` - libgtk-2-0 - libgconf-2-4 - xvfb (if you're running it headless) -## running +## setup +- `sudo apt install libgtk2.0-0 libgconf-2-4` (if you're on debian, other + distros idk) +- `sudo apt install xvfb` (again, on debian) +- `git clone https://tildegit.org/sose/sk0r` +- `cd sk0r` +- `npm install nightmare` +- `npm install http` +- `sudo npm install -g electron --unsafe-perm=true` - `node sk0r.js` - or `xvfb-run node sk0r.js` for headless -- it will output the scores as json to stdout +- the server will serve live game data in JSON format on localhost port 8000 + - not too sure about node's http server security, you might want to + use a reverse proxy if you're exposing it to the web + +## using +- use you favourite http client or web browser and point it to + 'http://localhost:8000' + - JSON will be served +- scores will update each time you access the page +- to refresh the electron browser page, access 'http://localhost:8000?reload' + - this will fetch the newest matches, but is generally only necessary if + they havent been refreshed in a while + - it also sends a request to hltv's servers, so use it sparingly + +## config +- if you want to change the host or port you can change the variables at the + top of the script +- to modify rate limiting behavior, you can also modify those variables at the + top of the script as well + +## why? +- its really hard to scrape hltv live scores any other way ## notes - its electron, don't run it on your 340e -- its very slow +- the rate limit functionality is not intuitive, but it was easy to write - it's named after [this kid](https://www.hltv.org/player/18638/sk0r) (hes good) ## additional notes @@ -32,7 +60,8 @@ a script to get live scores from hltv.org ## more notes - hltv doesn't like you scraping them, they'll probably break this and/or send me a c&d soon -- in keeping with the theme of naming projects after counter-strike players, my - next project will be a mobile phone operating system named "ANDROID" +- in keeping with the theme of naming projects after counter-strike players, + the next project i release will be a mobile phone operating system named +"ANDROID" (c) oneseveneight/sose diff --git a/sk0r.js b/sk0r.js index 22cb37b..4039dea 100644 --- a/sk0r.js +++ b/sk0r.js @@ -1,97 +1,158 @@ +const http = require("http"); const Nightmare = require('nightmare') -const nightmare = Nightmare({ show: true }) +const nightmare = Nightmare({ + show: false, +}) -var hltv_url = "https://hltv.org"; +const hltv_url = "https://hltv.org"; +const host = 'localhost'; +const port = 8000; -nightmare -.goto(hltv_url) -.wait("div[class='teamrows']") -.wait(2000) -.evaluate(function(hltv_url) { - var match_rows = document.querySelectorAll("div.teamrows"); - var matches = []; - for(var i = 0; i < match_rows.length; i++) { - var match_row = match_rows[i]; - var team_rows = match_row.children; - var parent = match_row.parentElement; - var grandparent = match_row.parentElement.parentElement; - var sibling = match_row.nextElementSibling; - // if (grandparent.className === "col-box a-reset") { continue; } +// note that rate limit only applies to reload requests, since they affect hltv's servers +const rate_limit_max_requests = 20; // maximum number of requests allow in the interval +const rate_limit_timer_dec_interval = 5000; // the lower the number the more aggressive the rate limit - // since the VOD boxes are always at the end we can just break - if (grandparent.className === "col-box a-reset") { break; } +function update_scores(hltv_url) { + var match_rows = document.querySelectorAll("div.teamrows"); + var matches = []; + for (var i = 0; i < match_rows.length; i++) { + var match_row = match_rows[i]; + var team_rows = match_row.children; + var parent = match_row.parentElement; + var grandparent = match_row.parentElement.parentElement; + var sibling = match_row.nextElementSibling; + // if (grandparent.className === "col-box a-reset") { continue; } - var event_name; - var start_time; - var link; - var stars; - var lan; - var team_names = []; - var team_countries = []; - var current_scores = []; - var maps_won = []; + // since the VOD boxes are always at the end we can just break here + if (grandparent.className === "col-box a-reset") { + break; + } + + var event_name; + var start_time; + var link; + var stars; + var lan; + var team_names = []; + var team_countries = []; + var current_scores = []; + var maps_won = []; - for (var j = 0; j < team_rows.length; j++){ - var team_row = team_rows[j]; - if (team_row.className.includes("teamrow")) { - var flag_el = team_row.getElementsByClassName("flag")[0]; - var team_el = team_row.getElementsByClassName("team")[0]; - team_countries.push(flag_el.getAttribute("title")); - team_names.push(team_el.innerText); - } + for (var j = 0; j < team_rows.length; j++) { + var team_row = team_rows[j]; + if (team_row.className.includes("teamrow")) { + var flag_el = team_row.getElementsByClassName("flag")[0]; + var team_el = team_row.getElementsByClassName("team")[0]; + if (typeof flag_el != 'undefined') { + team_countries.push(flag_el.getAttribute("title")); } + team_names.push(team_el.innerText); + } + } - if (grandparent.className.includes("hotmatch-box")) { - stars = parent.getAttribute("stars"); - lan = parent.getAttribute("lan"); - link = grandparent.getAttribute("href"); - event_name = grandparent.getAttribute("title") - start_time = "LIVE" - } else if (grandparent.className.includes("col-box-con result-box")) { - stars = grandparent.getAttribute("stars"); - lan = grandparent.getAttribute("lan"); - link = grandparent.getAttribute("href"); - event_name = parent.previousElementSibling.getAttribute("title"); - start_time = "OVER" + if (grandparent.className.includes("hotmatch-box")) { + stars = parent.getAttribute("stars"); + lan = parent.getAttribute("lan"); + link = grandparent.getAttribute("href"); + event_name = grandparent.getAttribute("title") + start_time = "LIVE" + } else if (grandparent.className.includes("col-box-con result-box")) { + stars = grandparent.getAttribute("stars"); + lan = grandparent.getAttribute("lan"); + link = grandparent.getAttribute("href"); + event_name = parent.previousElementSibling.getAttribute("title"); + start_time = "OVER" + } + + if (sibling.className === "twoRowExtra") { + score_rows = sibling.children; + for (var j = 0; j < score_rows.length; j++) { + var score_row = score_rows[j]; + if (score_row.className === "livescore twoRowExtraRow") { + var score_el = score_row.querySelector("[data-livescore-current-map-score='']"); + var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']"); + current_scores.push(score_el.innerText); + maps_won.push(maps_won_el.innerText); + } else if (score_row.className.includes("twoRowExtraRow won") || + score_row.className.includes("twoRowExtraRow lost")) { + maps_won.push(score_row.innerText); } + } + } else if (sibling.className === "middleExtra") { + start_time = sibling.getAttribute("data-unix"); + } - if (sibling.className === "twoRowExtra") { - score_rows = sibling.children; - for (var j = 0; j < score_rows.length; j++) { - var score_row = score_rows[j]; - if (score_row.className === "livescore twoRowExtraRow") { - var score_el = score_row.querySelector("[data-livescore-current-map-score='']"); - var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']"); - current_scores.push(score_el.innerText); - maps_won.push(maps_won_el.innerText); - } else if (score_row.className.includes("twoRowExtraRow won") - || score_row.className.includes("twoRowExtraRow lost")) { - maps_won.push(score_row.innerText); - } - } - } else if (sibling.className === "middleExtra") { - start_time = sibling.getAttribute("data-unix"); - } + if (link !== null) { + link = hltv_url + link; + } - link = hltv_url + link; + matches.push({ + "event_name": event_name, + "start_time": start_time, + "link": link, + "stars": stars, + "lan": lan, + "team_names": team_names, + "team_countries": team_countries, + "current_scores": current_scores, + "maps_won": maps_won + }); - matches.push({"event_name": event_name, - "start_time": start_time, - "link": link, - "stars": stars, - "lan": lan, - "team_names": team_names, - "team_countries": team_countries, - "current_scores": current_scores, - "maps_won": maps_won}); + } + var current_time = new Date().getTime(); + matches.unshift({ + "updated_on": current_time + }); + return JSON.stringify(matches); +} - } - return JSON.stringify(matches); - }, hltv_url) -.end() -.then(m => console.log(m)) -.then(nightmare.end()) -.catch(error => { - console.error(error) -}); + +async function run(hltv_url, host, port) { + var rate_limit_timer = 0; + var on_cooldown = false; + setInterval(() => { + if (rate_limit_timer > 0) { + rate_limit_timer--; + } else { + on_cooldown = false; + } + }, rate_limit_timer_dec_interval); + + const hltv = nightmare + .goto(hltv_url) + .wait("div[class='teamrows']") + .wait(2000); + + const request_handler = async function(request, response) { + response.writeHead(200, { + "Content-Type": "application/json; charset=UTF-8" + }); + const query_string = request.url.split("?").slice(-1)[0]; + if (query_string === "reload") { + if (rate_limit_timer >= rate_limit_max_requests) { + on_cooldown = true; + } + if (on_cooldown === false) { + await hltv.refresh(); + rate_limit_timer++; + } + } + await hltv.evaluate(update_scores, hltv_url) + .then((matches) => { + response.write(matches); + response.end(); + }) + .catch(error => { + response.write(error); + response.end(); + }); + } + + const server = http.createServer(request_handler); + server.listen(port, host, () => { + console.log(`Server is running on ${host}:${port}`); + }); +} + +run(hltv_url, host, port);