some changes
This commit is contained in:
parent
fe1414e5dc
commit
d1757f48bf
|
@ -1,3 +1,6 @@
|
||||||
node_modules/
|
node_modules/
|
||||||
package-lock.json
|
package-lock.json
|
||||||
skor
|
skor
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*.swn
|
||||||
|
|
47
readme.md
47
readme.md
|
@ -3,25 +3,53 @@
|
||||||
|
|
||||||
a script to get live scores from hltv.org
|
a script to get live scores from hltv.org
|
||||||
|
|
||||||
## why?
|
|
||||||
- its really hard to scrape hltv live scores (i tried)
|
|
||||||
|
|
||||||
## requirements
|
## requirements
|
||||||
|
- node.js
|
||||||
|
- npm
|
||||||
|
- `http` node package
|
||||||
- nightmare
|
- nightmare
|
||||||
- electron
|
- electron
|
||||||
- if npm is giving you perm errors try `sudo npm install -g electron --unsafe-perm=true`
|
|
||||||
- libgtk-2-0
|
- libgtk-2-0
|
||||||
- libgconf-2-4
|
- libgconf-2-4
|
||||||
- xvfb (if you're running it headless)
|
- xvfb (if you're running it headless)
|
||||||
|
|
||||||
## running
|
## setup
|
||||||
|
- `sudo apt install libgtk2.0-0 libgconf-2-4` (if you're on debian, other
|
||||||
|
distros idk)
|
||||||
|
- `sudo apt install xvfb` (again, on debian)
|
||||||
|
- `git clone https://tildegit.org/sose/sk0r`
|
||||||
|
- `cd sk0r`
|
||||||
|
- `npm install nightmare`
|
||||||
|
- `npm install http`
|
||||||
|
- `sudo npm install -g electron --unsafe-perm=true`
|
||||||
- `node sk0r.js`
|
- `node sk0r.js`
|
||||||
- or `xvfb-run node sk0r.js` for headless
|
- or `xvfb-run node sk0r.js` for headless
|
||||||
- it will output the scores as json to stdout
|
- the server will serve live game data in JSON format on localhost port 8000
|
||||||
|
- not too sure about node's http server security, you might want to
|
||||||
|
use a reverse proxy if you're exposing it to the web
|
||||||
|
|
||||||
|
## using
|
||||||
|
- use you favourite http client or web browser and point it to
|
||||||
|
'http://localhost:8000'
|
||||||
|
- JSON will be served
|
||||||
|
- scores will update each time you access the page
|
||||||
|
- to refresh the electron browser page, access 'http://localhost:8000?reload'
|
||||||
|
- this will fetch the newest matches, but is generally only necessary if
|
||||||
|
they havent been refreshed in a while
|
||||||
|
- it also sends a request to hltv's servers, so use it sparingly
|
||||||
|
|
||||||
|
## config
|
||||||
|
- if you want to change the host or port you can change the variables at the
|
||||||
|
top of the script
|
||||||
|
- to modify rate limiting behavior, you can also modify those variables at the
|
||||||
|
top of the script as well
|
||||||
|
|
||||||
|
## why?
|
||||||
|
- its really hard to scrape hltv live scores any other way
|
||||||
|
|
||||||
## notes
|
## notes
|
||||||
- its electron, don't run it on your 340e
|
- its electron, don't run it on your 340e
|
||||||
- its very slow
|
- the rate limit functionality is not intuitive, but it was easy to write
|
||||||
- it's named after [this kid](https://www.hltv.org/player/18638/sk0r) (hes good)
|
- it's named after [this kid](https://www.hltv.org/player/18638/sk0r) (hes good)
|
||||||
|
|
||||||
## additional notes
|
## additional notes
|
||||||
|
@ -32,7 +60,8 @@ a script to get live scores from hltv.org
|
||||||
## more notes
|
## more notes
|
||||||
- hltv doesn't like you scraping them, they'll probably break this and/or send
|
- hltv doesn't like you scraping them, they'll probably break this and/or send
|
||||||
me a c&d soon
|
me a c&d soon
|
||||||
- in keeping with the theme of naming projects after counter-strike players, my
|
- in keeping with the theme of naming projects after counter-strike players,
|
||||||
next project will be a mobile phone operating system named "ANDROID"
|
the next project i release will be a mobile phone operating system named
|
||||||
|
"ANDROID"
|
||||||
|
|
||||||
(c) oneseveneight/sose
|
(c) oneseveneight/sose
|
||||||
|
|
227
sk0r.js
227
sk0r.js
|
@ -1,97 +1,158 @@
|
||||||
|
const http = require("http");
|
||||||
const Nightmare = require('nightmare')
|
const Nightmare = require('nightmare')
|
||||||
const nightmare = Nightmare({ show: true })
|
const nightmare = Nightmare({
|
||||||
|
show: false,
|
||||||
|
})
|
||||||
|
|
||||||
var hltv_url = "https://hltv.org";
|
const hltv_url = "https://hltv.org";
|
||||||
|
const host = 'localhost';
|
||||||
|
const port = 8000;
|
||||||
|
|
||||||
nightmare
|
// note that rate limit only applies to reload requests, since they affect hltv's servers
|
||||||
.goto(hltv_url)
|
const rate_limit_max_requests = 20; // maximum number of requests allow in the interval
|
||||||
.wait("div[class='teamrows']")
|
const rate_limit_timer_dec_interval = 5000; // the lower the number the more aggressive the rate limit
|
||||||
.wait(2000)
|
|
||||||
.evaluate(function(hltv_url) {
|
|
||||||
var match_rows = document.querySelectorAll("div.teamrows");
|
|
||||||
var matches = [];
|
|
||||||
for(var i = 0; i < match_rows.length; i++) {
|
|
||||||
var match_row = match_rows[i];
|
|
||||||
var team_rows = match_row.children;
|
|
||||||
var parent = match_row.parentElement;
|
|
||||||
var grandparent = match_row.parentElement.parentElement;
|
|
||||||
var sibling = match_row.nextElementSibling;
|
|
||||||
// if (grandparent.className === "col-box a-reset") { continue; }
|
|
||||||
|
|
||||||
// since the VOD boxes are always at the end we can just break
|
function update_scores(hltv_url) {
|
||||||
if (grandparent.className === "col-box a-reset") { break; }
|
var match_rows = document.querySelectorAll("div.teamrows");
|
||||||
|
var matches = [];
|
||||||
|
for (var i = 0; i < match_rows.length; i++) {
|
||||||
|
var match_row = match_rows[i];
|
||||||
|
var team_rows = match_row.children;
|
||||||
|
var parent = match_row.parentElement;
|
||||||
|
var grandparent = match_row.parentElement.parentElement;
|
||||||
|
var sibling = match_row.nextElementSibling;
|
||||||
|
// if (grandparent.className === "col-box a-reset") { continue; }
|
||||||
|
|
||||||
var event_name;
|
// since the VOD boxes are always at the end we can just break here
|
||||||
var start_time;
|
if (grandparent.className === "col-box a-reset") {
|
||||||
var link;
|
break;
|
||||||
var stars;
|
}
|
||||||
var lan;
|
|
||||||
var team_names = [];
|
var event_name;
|
||||||
var team_countries = [];
|
var start_time;
|
||||||
var current_scores = [];
|
var link;
|
||||||
var maps_won = [];
|
var stars;
|
||||||
|
var lan;
|
||||||
|
var team_names = [];
|
||||||
|
var team_countries = [];
|
||||||
|
var current_scores = [];
|
||||||
|
var maps_won = [];
|
||||||
|
|
||||||
|
|
||||||
for (var j = 0; j < team_rows.length; j++){
|
for (var j = 0; j < team_rows.length; j++) {
|
||||||
var team_row = team_rows[j];
|
var team_row = team_rows[j];
|
||||||
if (team_row.className.includes("teamrow")) {
|
if (team_row.className.includes("teamrow")) {
|
||||||
var flag_el = team_row.getElementsByClassName("flag")[0];
|
var flag_el = team_row.getElementsByClassName("flag")[0];
|
||||||
var team_el = team_row.getElementsByClassName("team")[0];
|
var team_el = team_row.getElementsByClassName("team")[0];
|
||||||
team_countries.push(flag_el.getAttribute("title"));
|
if (typeof flag_el != 'undefined') {
|
||||||
team_names.push(team_el.innerText);
|
team_countries.push(flag_el.getAttribute("title"));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
team_names.push(team_el.innerText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (grandparent.className.includes("hotmatch-box")) {
|
if (grandparent.className.includes("hotmatch-box")) {
|
||||||
stars = parent.getAttribute("stars");
|
stars = parent.getAttribute("stars");
|
||||||
lan = parent.getAttribute("lan");
|
lan = parent.getAttribute("lan");
|
||||||
link = grandparent.getAttribute("href");
|
link = grandparent.getAttribute("href");
|
||||||
event_name = grandparent.getAttribute("title")
|
event_name = grandparent.getAttribute("title")
|
||||||
start_time = "LIVE"
|
start_time = "LIVE"
|
||||||
} else if (grandparent.className.includes("col-box-con result-box")) {
|
} else if (grandparent.className.includes("col-box-con result-box")) {
|
||||||
stars = grandparent.getAttribute("stars");
|
stars = grandparent.getAttribute("stars");
|
||||||
lan = grandparent.getAttribute("lan");
|
lan = grandparent.getAttribute("lan");
|
||||||
link = grandparent.getAttribute("href");
|
link = grandparent.getAttribute("href");
|
||||||
event_name = parent.previousElementSibling.getAttribute("title");
|
event_name = parent.previousElementSibling.getAttribute("title");
|
||||||
start_time = "OVER"
|
start_time = "OVER"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sibling.className === "twoRowExtra") {
|
||||||
|
score_rows = sibling.children;
|
||||||
|
for (var j = 0; j < score_rows.length; j++) {
|
||||||
|
var score_row = score_rows[j];
|
||||||
|
if (score_row.className === "livescore twoRowExtraRow") {
|
||||||
|
var score_el = score_row.querySelector("[data-livescore-current-map-score='']");
|
||||||
|
var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']");
|
||||||
|
current_scores.push(score_el.innerText);
|
||||||
|
maps_won.push(maps_won_el.innerText);
|
||||||
|
} else if (score_row.className.includes("twoRowExtraRow won") ||
|
||||||
|
score_row.className.includes("twoRowExtraRow lost")) {
|
||||||
|
maps_won.push(score_row.innerText);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
} else if (sibling.className === "middleExtra") {
|
||||||
|
start_time = sibling.getAttribute("data-unix");
|
||||||
|
}
|
||||||
|
|
||||||
if (sibling.className === "twoRowExtra") {
|
if (link !== null) {
|
||||||
score_rows = sibling.children;
|
link = hltv_url + link;
|
||||||
for (var j = 0; j < score_rows.length; j++) {
|
}
|
||||||
var score_row = score_rows[j];
|
|
||||||
if (score_row.className === "livescore twoRowExtraRow") {
|
|
||||||
var score_el = score_row.querySelector("[data-livescore-current-map-score='']");
|
|
||||||
var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']");
|
|
||||||
current_scores.push(score_el.innerText);
|
|
||||||
maps_won.push(maps_won_el.innerText);
|
|
||||||
} else if (score_row.className.includes("twoRowExtraRow won")
|
|
||||||
|| score_row.className.includes("twoRowExtraRow lost")) {
|
|
||||||
maps_won.push(score_row.innerText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (sibling.className === "middleExtra") {
|
|
||||||
start_time = sibling.getAttribute("data-unix");
|
|
||||||
}
|
|
||||||
|
|
||||||
link = hltv_url + link;
|
matches.push({
|
||||||
|
"event_name": event_name,
|
||||||
|
"start_time": start_time,
|
||||||
|
"link": link,
|
||||||
|
"stars": stars,
|
||||||
|
"lan": lan,
|
||||||
|
"team_names": team_names,
|
||||||
|
"team_countries": team_countries,
|
||||||
|
"current_scores": current_scores,
|
||||||
|
"maps_won": maps_won
|
||||||
|
});
|
||||||
|
|
||||||
matches.push({"event_name": event_name,
|
}
|
||||||
"start_time": start_time,
|
var current_time = new Date().getTime();
|
||||||
"link": link,
|
matches.unshift({
|
||||||
"stars": stars,
|
"updated_on": current_time
|
||||||
"lan": lan,
|
});
|
||||||
"team_names": team_names,
|
return JSON.stringify(matches);
|
||||||
"team_countries": team_countries,
|
}
|
||||||
"current_scores": current_scores,
|
|
||||||
"maps_won": maps_won});
|
|
||||||
|
|
||||||
}
|
|
||||||
return JSON.stringify(matches);
|
async function run(hltv_url, host, port) {
|
||||||
}, hltv_url)
|
var rate_limit_timer = 0;
|
||||||
.end()
|
var on_cooldown = false;
|
||||||
.then(m => console.log(m))
|
setInterval(() => {
|
||||||
.then(nightmare.end())
|
if (rate_limit_timer > 0) {
|
||||||
.catch(error => {
|
rate_limit_timer--;
|
||||||
console.error(error)
|
} else {
|
||||||
});
|
on_cooldown = false;
|
||||||
|
}
|
||||||
|
}, rate_limit_timer_dec_interval);
|
||||||
|
|
||||||
|
const hltv = nightmare
|
||||||
|
.goto(hltv_url)
|
||||||
|
.wait("div[class='teamrows']")
|
||||||
|
.wait(2000);
|
||||||
|
|
||||||
|
const request_handler = async function(request, response) {
|
||||||
|
response.writeHead(200, {
|
||||||
|
"Content-Type": "application/json; charset=UTF-8"
|
||||||
|
});
|
||||||
|
const query_string = request.url.split("?").slice(-1)[0];
|
||||||
|
if (query_string === "reload") {
|
||||||
|
if (rate_limit_timer >= rate_limit_max_requests) {
|
||||||
|
on_cooldown = true;
|
||||||
|
}
|
||||||
|
if (on_cooldown === false) {
|
||||||
|
await hltv.refresh();
|
||||||
|
rate_limit_timer++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await hltv.evaluate(update_scores, hltv_url)
|
||||||
|
.then((matches) => {
|
||||||
|
response.write(matches);
|
||||||
|
response.end();
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
response.write(error);
|
||||||
|
response.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const server = http.createServer(request_handler);
|
||||||
|
server.listen(port, host, () => {
|
||||||
|
console.log(`Server is running on ${host}:${port}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
run(hltv_url, host, port);
|
||||||
|
|
Loading…
Reference in New Issue