some changes
This commit is contained in:
parent
fe1414e5dc
commit
d1757f48bf
|
|
@ -1,3 +1,6 @@
|
|||
node_modules/
|
||||
package-lock.json
|
||||
skor
|
||||
*.swp
|
||||
*.swo
|
||||
*.swn
|
||||
|
|
|
|||
47
readme.md
47
readme.md
|
|
@ -3,25 +3,53 @@
|
|||
|
||||
a script to get live scores from hltv.org
|
||||
|
||||
## why?
|
||||
- its really hard to scrape hltv live scores (i tried)
|
||||
|
||||
## requirements
|
||||
- node.js
|
||||
- npm
|
||||
- `http` node package
|
||||
- nightmare
|
||||
- electron
|
||||
- if npm is giving you perm errors try `sudo npm install -g electron --unsafe-perm=true`
|
||||
- libgtk-2-0
|
||||
- libgconf-2-4
|
||||
- xvfb (if you're running it headless)
|
||||
|
||||
## running
|
||||
## setup
|
||||
- `sudo apt install libgtk2.0-0 libgconf-2-4` (if you're on debian, other
|
||||
distros idk)
|
||||
- `sudo apt install xvfb` (again, on debian)
|
||||
- `git clone https://tildegit.org/sose/sk0r`
|
||||
- `cd sk0r`
|
||||
- `npm install nightmare`
|
||||
- `npm install http`
|
||||
- `sudo npm install -g electron --unsafe-perm=true`
|
||||
- `node sk0r.js`
|
||||
- or `xvfb-run node sk0r.js` for headless
|
||||
- it will output the scores as json to stdout
|
||||
- the server will serve live game data in JSON format on localhost port 8000
|
||||
- not too sure about node's http server security, you might want to
|
||||
use a reverse proxy if you're exposing it to the web
|
||||
|
||||
## using
|
||||
- use you favourite http client or web browser and point it to
|
||||
'http://localhost:8000'
|
||||
- JSON will be served
|
||||
- scores will update each time you access the page
|
||||
- to refresh the electron browser page, access 'http://localhost:8000?reload'
|
||||
- this will fetch the newest matches, but is generally only necessary if
|
||||
they havent been refreshed in a while
|
||||
- it also sends a request to hltv's servers, so use it sparingly
|
||||
|
||||
## config
|
||||
- if you want to change the host or port you can change the variables at the
|
||||
top of the script
|
||||
- to modify rate limiting behavior, you can also modify those variables at the
|
||||
top of the script as well
|
||||
|
||||
## why?
|
||||
- its really hard to scrape hltv live scores any other way
|
||||
|
||||
## notes
|
||||
- its electron, don't run it on your 340e
|
||||
- its very slow
|
||||
- the rate limit functionality is not intuitive, but it was easy to write
|
||||
- it's named after [this kid](https://www.hltv.org/player/18638/sk0r) (hes good)
|
||||
|
||||
## additional notes
|
||||
|
|
@ -32,7 +60,8 @@ a script to get live scores from hltv.org
|
|||
## more notes
|
||||
- hltv doesn't like you scraping them, they'll probably break this and/or send
|
||||
me a c&d soon
|
||||
- in keeping with the theme of naming projects after counter-strike players, my
|
||||
next project will be a mobile phone operating system named "ANDROID"
|
||||
- in keeping with the theme of naming projects after counter-strike players,
|
||||
the next project i release will be a mobile phone operating system named
|
||||
"ANDROID"
|
||||
|
||||
(c) oneseveneight/sose
|
||||
|
|
|
|||
227
sk0r.js
227
sk0r.js
|
|
@ -1,97 +1,158 @@
|
|||
const http = require("http");
|
||||
const Nightmare = require('nightmare')
|
||||
const nightmare = Nightmare({ show: true })
|
||||
const nightmare = Nightmare({
|
||||
show: false,
|
||||
})
|
||||
|
||||
var hltv_url = "https://hltv.org";
|
||||
const hltv_url = "https://hltv.org";
|
||||
const host = 'localhost';
|
||||
const port = 8000;
|
||||
|
||||
nightmare
|
||||
.goto(hltv_url)
|
||||
.wait("div[class='teamrows']")
|
||||
.wait(2000)
|
||||
.evaluate(function(hltv_url) {
|
||||
var match_rows = document.querySelectorAll("div.teamrows");
|
||||
var matches = [];
|
||||
for(var i = 0; i < match_rows.length; i++) {
|
||||
var match_row = match_rows[i];
|
||||
var team_rows = match_row.children;
|
||||
var parent = match_row.parentElement;
|
||||
var grandparent = match_row.parentElement.parentElement;
|
||||
var sibling = match_row.nextElementSibling;
|
||||
// if (grandparent.className === "col-box a-reset") { continue; }
|
||||
// note that rate limit only applies to reload requests, since they affect hltv's servers
|
||||
const rate_limit_max_requests = 20; // maximum number of requests allow in the interval
|
||||
const rate_limit_timer_dec_interval = 5000; // the lower the number the more aggressive the rate limit
|
||||
|
||||
// since the VOD boxes are always at the end we can just break
|
||||
if (grandparent.className === "col-box a-reset") { break; }
|
||||
function update_scores(hltv_url) {
|
||||
var match_rows = document.querySelectorAll("div.teamrows");
|
||||
var matches = [];
|
||||
for (var i = 0; i < match_rows.length; i++) {
|
||||
var match_row = match_rows[i];
|
||||
var team_rows = match_row.children;
|
||||
var parent = match_row.parentElement;
|
||||
var grandparent = match_row.parentElement.parentElement;
|
||||
var sibling = match_row.nextElementSibling;
|
||||
// if (grandparent.className === "col-box a-reset") { continue; }
|
||||
|
||||
var event_name;
|
||||
var start_time;
|
||||
var link;
|
||||
var stars;
|
||||
var lan;
|
||||
var team_names = [];
|
||||
var team_countries = [];
|
||||
var current_scores = [];
|
||||
var maps_won = [];
|
||||
// since the VOD boxes are always at the end we can just break here
|
||||
if (grandparent.className === "col-box a-reset") {
|
||||
break;
|
||||
}
|
||||
|
||||
var event_name;
|
||||
var start_time;
|
||||
var link;
|
||||
var stars;
|
||||
var lan;
|
||||
var team_names = [];
|
||||
var team_countries = [];
|
||||
var current_scores = [];
|
||||
var maps_won = [];
|
||||
|
||||
|
||||
for (var j = 0; j < team_rows.length; j++){
|
||||
var team_row = team_rows[j];
|
||||
if (team_row.className.includes("teamrow")) {
|
||||
var flag_el = team_row.getElementsByClassName("flag")[0];
|
||||
var team_el = team_row.getElementsByClassName("team")[0];
|
||||
team_countries.push(flag_el.getAttribute("title"));
|
||||
team_names.push(team_el.innerText);
|
||||
}
|
||||
for (var j = 0; j < team_rows.length; j++) {
|
||||
var team_row = team_rows[j];
|
||||
if (team_row.className.includes("teamrow")) {
|
||||
var flag_el = team_row.getElementsByClassName("flag")[0];
|
||||
var team_el = team_row.getElementsByClassName("team")[0];
|
||||
if (typeof flag_el != 'undefined') {
|
||||
team_countries.push(flag_el.getAttribute("title"));
|
||||
}
|
||||
team_names.push(team_el.innerText);
|
||||
}
|
||||
}
|
||||
|
||||
if (grandparent.className.includes("hotmatch-box")) {
|
||||
stars = parent.getAttribute("stars");
|
||||
lan = parent.getAttribute("lan");
|
||||
link = grandparent.getAttribute("href");
|
||||
event_name = grandparent.getAttribute("title")
|
||||
start_time = "LIVE"
|
||||
} else if (grandparent.className.includes("col-box-con result-box")) {
|
||||
stars = grandparent.getAttribute("stars");
|
||||
lan = grandparent.getAttribute("lan");
|
||||
link = grandparent.getAttribute("href");
|
||||
event_name = parent.previousElementSibling.getAttribute("title");
|
||||
start_time = "OVER"
|
||||
if (grandparent.className.includes("hotmatch-box")) {
|
||||
stars = parent.getAttribute("stars");
|
||||
lan = parent.getAttribute("lan");
|
||||
link = grandparent.getAttribute("href");
|
||||
event_name = grandparent.getAttribute("title")
|
||||
start_time = "LIVE"
|
||||
} else if (grandparent.className.includes("col-box-con result-box")) {
|
||||
stars = grandparent.getAttribute("stars");
|
||||
lan = grandparent.getAttribute("lan");
|
||||
link = grandparent.getAttribute("href");
|
||||
event_name = parent.previousElementSibling.getAttribute("title");
|
||||
start_time = "OVER"
|
||||
}
|
||||
|
||||
if (sibling.className === "twoRowExtra") {
|
||||
score_rows = sibling.children;
|
||||
for (var j = 0; j < score_rows.length; j++) {
|
||||
var score_row = score_rows[j];
|
||||
if (score_row.className === "livescore twoRowExtraRow") {
|
||||
var score_el = score_row.querySelector("[data-livescore-current-map-score='']");
|
||||
var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']");
|
||||
current_scores.push(score_el.innerText);
|
||||
maps_won.push(maps_won_el.innerText);
|
||||
} else if (score_row.className.includes("twoRowExtraRow won") ||
|
||||
score_row.className.includes("twoRowExtraRow lost")) {
|
||||
maps_won.push(score_row.innerText);
|
||||
}
|
||||
}
|
||||
} else if (sibling.className === "middleExtra") {
|
||||
start_time = sibling.getAttribute("data-unix");
|
||||
}
|
||||
|
||||
if (sibling.className === "twoRowExtra") {
|
||||
score_rows = sibling.children;
|
||||
for (var j = 0; j < score_rows.length; j++) {
|
||||
var score_row = score_rows[j];
|
||||
if (score_row.className === "livescore twoRowExtraRow") {
|
||||
var score_el = score_row.querySelector("[data-livescore-current-map-score='']");
|
||||
var maps_won_el = score_row.querySelector("[data-livescore-maps-won-for='']");
|
||||
current_scores.push(score_el.innerText);
|
||||
maps_won.push(maps_won_el.innerText);
|
||||
} else if (score_row.className.includes("twoRowExtraRow won")
|
||||
|| score_row.className.includes("twoRowExtraRow lost")) {
|
||||
maps_won.push(score_row.innerText);
|
||||
}
|
||||
}
|
||||
} else if (sibling.className === "middleExtra") {
|
||||
start_time = sibling.getAttribute("data-unix");
|
||||
}
|
||||
if (link !== null) {
|
||||
link = hltv_url + link;
|
||||
}
|
||||
|
||||
link = hltv_url + link;
|
||||
matches.push({
|
||||
"event_name": event_name,
|
||||
"start_time": start_time,
|
||||
"link": link,
|
||||
"stars": stars,
|
||||
"lan": lan,
|
||||
"team_names": team_names,
|
||||
"team_countries": team_countries,
|
||||
"current_scores": current_scores,
|
||||
"maps_won": maps_won
|
||||
});
|
||||
|
||||
matches.push({"event_name": event_name,
|
||||
"start_time": start_time,
|
||||
"link": link,
|
||||
"stars": stars,
|
||||
"lan": lan,
|
||||
"team_names": team_names,
|
||||
"team_countries": team_countries,
|
||||
"current_scores": current_scores,
|
||||
"maps_won": maps_won});
|
||||
}
|
||||
var current_time = new Date().getTime();
|
||||
matches.unshift({
|
||||
"updated_on": current_time
|
||||
});
|
||||
return JSON.stringify(matches);
|
||||
}
|
||||
|
||||
}
|
||||
return JSON.stringify(matches);
|
||||
}, hltv_url)
|
||||
.end()
|
||||
.then(m => console.log(m))
|
||||
.then(nightmare.end())
|
||||
.catch(error => {
|
||||
console.error(error)
|
||||
});
|
||||
|
||||
async function run(hltv_url, host, port) {
|
||||
var rate_limit_timer = 0;
|
||||
var on_cooldown = false;
|
||||
setInterval(() => {
|
||||
if (rate_limit_timer > 0) {
|
||||
rate_limit_timer--;
|
||||
} else {
|
||||
on_cooldown = false;
|
||||
}
|
||||
}, rate_limit_timer_dec_interval);
|
||||
|
||||
const hltv = nightmare
|
||||
.goto(hltv_url)
|
||||
.wait("div[class='teamrows']")
|
||||
.wait(2000);
|
||||
|
||||
const request_handler = async function(request, response) {
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "application/json; charset=UTF-8"
|
||||
});
|
||||
const query_string = request.url.split("?").slice(-1)[0];
|
||||
if (query_string === "reload") {
|
||||
if (rate_limit_timer >= rate_limit_max_requests) {
|
||||
on_cooldown = true;
|
||||
}
|
||||
if (on_cooldown === false) {
|
||||
await hltv.refresh();
|
||||
rate_limit_timer++;
|
||||
}
|
||||
}
|
||||
await hltv.evaluate(update_scores, hltv_url)
|
||||
.then((matches) => {
|
||||
response.write(matches);
|
||||
response.end();
|
||||
})
|
||||
.catch(error => {
|
||||
response.write(error);
|
||||
response.end();
|
||||
});
|
||||
}
|
||||
|
||||
const server = http.createServer(request_handler);
|
||||
server.listen(port, host, () => {
|
||||
console.log(`Server is running on ${host}:${port}`);
|
||||
});
|
||||
}
|
||||
|
||||
run(hltv_url, host, port);
|
||||
|
|
|
|||
Loading…
Reference in New Issue