39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
import requests
|
|
import re
|
|
import psycopg2
|
|
import psycopg2.extras
|
|
from datetime import datetime
|
|
from base64 import b64decode
|
|
import yaml
|
|
|
|
config = yaml.full_load(open("config.yaml").read())
|
|
pgconn = psycopg2.connect(dbname=config["dbname"], user=config["dbuser"], password=config["dbpassword"], host=config["dbhost"])
|
|
r = requests.get("http://www.vpngate.net/api/iphone/", headers={"User-Agent": config["user_agent"]})
|
|
csvfile = r.text.split("\r\n")
|
|
scraped_servers = []
|
|
current_servers = []
|
|
with pgconn:
|
|
with pgconn.cursor() as curs:
|
|
curs.execute("select ip, port from proxies where proxy_type='vpngate'")
|
|
current_servers = curs.fetchall()
|
|
|
|
for server in csvfile[2:]:
|
|
try:
|
|
hostname, ip, score, ping, speed, clong, cshort, numsessions, uptime, totalusers, totaltraffic, logtype, operator, message, configdata = server.split(",")
|
|
except:
|
|
continue
|
|
configdataplain = b64decode(configdata)
|
|
remote = re.search("remote (\S+) (\d+)", configdataplain.decode())
|
|
if remote:
|
|
host = remote[1]
|
|
port = int(remote[2])
|
|
if (host, port) not in current_servers:
|
|
scraped_servers.append((host, port, host, "vpngate", datetime.now(), "vpngate proxies api"))
|
|
|
|
with pgconn:
|
|
with pgconn.cursor() as curs:
|
|
insert_query = """
|
|
insert into proxies (ip, port, exit_ip, proxy_type, submitted_at, comment)
|
|
values %s"""
|
|
psycopg2.extras.execute_values(curs, insert_query, scraped_servers)
|