commit 4277b59ac5d79950415e22554bbca52115ed985d Author: khuxkm fbexl Date: Wed Apr 14 13:07:33 2021 +0000 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f203d1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,159 @@ +# password file +.password + +# index +_index/* + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +# .env +.env/ +.venv/ +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# operating system-related files +# file properties cache/storage on macOS +*.DS_Store +# thumbnail cache on Windows +Thumbs.db + +# profiling data +.prof + + +# End of https://www.toptal.com/developers/gitignore/api/python diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9077cd --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# teamwikibot + +A wiki search bot for tilde.team. diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..c0cb20a --- /dev/null +++ b/bot.py @@ -0,0 +1,178 @@ +import impmod, socket, traceback, events, time, os +from irc.client import NickMask +class Socket: + def __init__(self,server): + self.sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) + self.sock.connect(server) + self._read_buffer=b"" + def read(self): + try: + data=self.sock.recv(4096) + if not data: + return None + except: return traceback.print_exc() + data = self._read_buffer+data + self._read_buffer=b"" + lines = [line.strip(b"\r") for line in data.split(b"\n")] + if lines[-1]: + self._read_buffer=lines[-1] + lines.pop(-1) + lines = [line.decode("utf-8") for line in lines] + return lines + def send(self,line): + self.sock.send(line.encode("utf-8")) + def close(self): + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + +def unescape(value): + return value.replace(r"\:",";").replace(r"\s"," ").replace(r"\\","\\").replace(r"\r","\r").replace(r"\n","\n") + +def escape(value): + return value.replace(";",r"\:").replace(" ",r"\s").replace("\\",r"\\").replace("\r",r"\r").replace("\n",r"\n") + +MISSING = None + +class IRCLine: + def __init__(self,command,*params,tags=dict(),hostmask=""): + self.command=command + if len(params)==0: + self.params=[] + elif len(params)==1 and type(params[0]) in (list,tuple): + self.params=list(params[0]) + else: + self.params=list(params) + self.tags=tags + self.hostmask=NickMask(hostmask) if hostmask else None + @property + def line(self): + prefix="" + if len(list(self.tags.keys()))>0: + tagc = len(list(self.tags.keys())) + prefix+="@" + for i,tag in enumerate(self.tags.keys()): + prefix+=tag + if self.tags[tag] is not MISSING: + prefix+="="+escape(str(self.tags[tag])) + if (i+1)0 and not parts[i].startswith(":"): i-=1 + if i!=0: parts[i:]=[" ".join(parts[i:])] + return cls(*parts,tags=tags,hostmask=hostmask) + def encode(self,*args,**kwargs): + # clearly, if we're here, I'm an idiot and am trying to send an + # IRCLine object down the tube. just do it. + return self.line.encode(*args,**kwargs) + +PLUGIN_MODULES={} +class IRCBot: + def __init__(self,nickname,username,realname="IRCBot",server=("localhost",6667),channels=["#bots"]): + self.nickname=nickname + self.username=username + self.realname=realname + self.server=server + self.channels=channels + self.event_manager=events.EventManager() + self.in_batch=False + def load_modules(self): + self.event_manager.clear() + for name in os.listdir("plugins"): + if name.endswith(".py"): + self.load_module(name[:-3],os.path.join("plugins",name)) + def load_module(self,modname,path): + try: + if modname in PLUGIN_MODULES: + print("{} already imported, reloading".format(modname)) + PLUGIN_MODULES[modname].reload() + else: + try: + print("importing {}".format(modname)) + PLUGIN_MODULES[modname]=impmod.Module(modname,path) + except: + print("Unable to load plugin {}".format(modname)) + traceback.print_exc() + register_func = getattr(PLUGIN_MODULES[modname].module,"register",None) + if not register_func: + print(f"Plugin {modname} has no register function!") + print("Remember, if porting plugins from a minerbot-based architecture,") + print("you have to add a register function to use the new system.") + return + register_func(self) + except: + traceback.print_exc() + pass + def handle_line(self,line): + if type(line)!=IRCLine: line = IRCLine.parse_line(line) + self.event_manager(events.Event("raw_line",text=line.line,parsed=line)) + if line.command=="PING": + line.command="PONG" + self.socket.send(line.line) + return + if line.command=="BATCH": + self.in_batch = line.params[0].startswith("+") + if self.in_batch: + self.batch_reference_tag = line.params[0][1:] + self.batch_type = line.params[1] + self.batch_params = line.params[2:] + self.event_manager(events.Event("start_batch",reference_tag=self.batch_reference_tag,type=self.batch_type,params=self.batch_params)) + else: + self.batch_reference_tag = None + self.batch_type=None + self.batch_params=[] + self.event_manager(events.Event("end_batch",reference_tag=line.params[0][1:])) + if line.hostmask is None: return + if line.hostmask.nick==self.nickname: + return + if line.command in "PRIVMSG NOTICE".split(): + target = line.params[0] + message = line.params[1][1:] + self.event_manager(events.Event(line.command.lower(),target=target,message=message,tags=line.tags,hostmask=line.hostmask)) + elif line.command == "TAGMSG": + self.event_manager(events.Event("tagmsg",hostmask=line.hostmask,tags=line.tags,target=line.params[0])) + elif line.command == "INVITE": + self.event_manager(events.Event("invite",to=line.params[1][1:],hostmask=line.hostmask)) + elif line.command == "PING": + self.socket.send(IRCLine("PONG",line.params).line) + def start(self): + self.socket = Socket(self.server) + self.socket.send("NICK {}\r\n".format(self.nickname)) + self.socket.send("USER {} * * :{}\r\n".format(self.username,self.realname)) + time.sleep(2) # give the server some time to record my username + self.event_manager(events.Event("connection_established")) + for channel in self.channels: + self.socket.send(f"JOIN {channel}\r\n") + time.sleep(1) + self.socket.send("CAP REQ :message-tags batch\r\n") + self.running=True + while self.running: + lines = self.socket.read() + if lines: + for line in lines: + self.handle_line(line) + self.socket.close() + del self.socket + +if __name__=="__main__": + bot = IRCBot("teamwikibot","teamwikibot",channels=["#team"]) + bot.load_modules() + bot.start() diff --git a/create_index.py b/create_index.py new file mode 100644 index 0000000..34cfa86 --- /dev/null +++ b/create_index.py @@ -0,0 +1,33 @@ +from whoosh.fields import Schema, ID, TEXT, NUMERIC +from whoosh import index +from bs4 import BeautifulSoup +from markdown import markdown +import os, os.path, frontmatter +import sys + +def textify(md): + return BeautifulSoup(markdown(md),"lxml").text + +schema = Schema( + url = ID(stored=True), + title = ID(stored=True), + text = TEXT +) + +if not os.path.exists("_index"): + os.mkdir("_index") + +_index = index.create_in("_index",schema) +writer = _index.writer() + +pages = os.listdir("/var/www/tilde.team/wiki/pages") + +for page in pages: + with open(os.path.join("/var/www/tilde.team/wiki/pages",page)) as f: + post = frontmatter.load(f) + url = "https://tilde.wiki/"+os.path.splitext(page)[0] + title = post["title"] + text = textify(post.content) + writer.add_document(title=title, text=text, url=url) + +writer.commit(optimize=True) diff --git a/events.py b/events.py new file mode 100644 index 0000000..4665152 --- /dev/null +++ b/events.py @@ -0,0 +1,23 @@ +from collections import defaultdict + +class Event: + def __init__(self,name,**kwargs): + self.data=kwargs + self.name=name + def __getitem__(self,k): + return self.data[k] + def __getattr__(self,k): + if k in self.data: return self.data[k] + +class EventManager: + def __init__(self): + self.handlers=defaultdict(list) + def clear(self): + self.__init__() + def on(self,event,func): + self.handlers[event].append(func) + def __call__(self,event_obj): + print(event_obj.name,event_obj.data) + if event_obj.name not in self.handlers: return + handlers = self.handlers[event_obj.name] + for handler in handlers: handler(event_obj) diff --git a/impmod.py b/impmod.py new file mode 100644 index 0000000..db430cb --- /dev/null +++ b/impmod.py @@ -0,0 +1,19 @@ +import importlib, importlib.util, sys + +class Module: + """A module. Stores module object, spec object, and handles reloading.""" + def __init__(self,modname,path=None): + if path is None: + path = modname+".py" + self.modname, self.path = modname, path + self.spec = importlib.util.spec_from_file_location(modname,path) + self.module = importlib.util.module_from_spec(self.spec) + self.spec.loader.exec_module(self.module) + def reload(self): + if self.modname not in sys.modules: + sys.modules[self.modname]=self.module + # Alright, this needs some explaining. + # When you do importlib.reload, it does some juju magic shist to find the spec and calls importlib._bootstrap._exec. + # When you dynamically import a module it won't do its magic correctly and it'll error. + # Luckily, we can skip all the juju magic since we can just store the spec. + importlib._bootstrap._exec(self.spec,self.module) diff --git a/plugins/admin.py b/plugins/admin.py new file mode 100644 index 0000000..a504a99 --- /dev/null +++ b/plugins/admin.py @@ -0,0 +1,46 @@ +import importlib, events +importlib.reload(events) +from events import Event +from bot import IRCLine +ADMIN_HOSTMASKS = ["khuxkm!khuxkm@fuckup.club","ben!ben@oper.tilde.chat"] +BOT = None + +def admin(event): + if BOT is None: return + if event.hostmask not in ADMIN_HOSTMASKS: + BOT.socket.send(IRCLine("PRIVMSG",event.target if event.target.startswith("#") else event.hostmask.nick,"You're not the boss of me! (hostmask {!s})".format(event.hostmask)).line) + return + if len(event.parts)==0: return + if event.parts[0]=="reload": + BOT.load_modules() + elif event.parts[0]=="quit": + BOT.socket.send(IRCLine("QUIT",[":goodbye"]).line) + BOT.running=False + elif event.parts[0]=="msg": + target = event.parts[1] + message = ":"+" ".join(event.parts[2:]) + BOT.socket.send(IRCLine("PRIVMSG",target,message).line) + else: + event_out = Event("admin_"+event.parts[0]) + event_out.data.update(event.data) + event_out.data["parts"]=event.parts[1:] + BOT.event_manager(event_out) + +def on_invite(event): + if BOT is None: return + if event.hostmask not in ADMIN_HOSTMASKS: return + BOT.socket.send(IRCLine("JOIN",[event.to]).line) + +PASSWORD="whoops" +try: + with open(".password") as f: PASSWORD=f.read().strip() +except: pass +def login(event): + BOT.socket.send(IRCLine("NS","IDENTIFY",PASSWORD).line) + +def register(bot): + global BOT + BOT=bot + bot.event_manager.on("command_admin",admin) + bot.event_manager.on("invite",on_invite) + bot.event_manager.on("connection_established",login) diff --git a/plugins/commands.py b/plugins/commands.py new file mode 100644 index 0000000..4a005c0 --- /dev/null +++ b/plugins/commands.py @@ -0,0 +1,19 @@ +import events +BOT=None + +def on_privmsg(event): + if BOT is None: return + if BOT.in_batch: return + prefix = BOT.prefix if event.target.startswith("#") else "" + if event.message.startswith(prefix): + parts = event.message.split(" ") + parts[0]=parts[0][len(prefix):] + event_out = events.Event("command_"+parts.pop(0),parts=parts) + event_out.data.update(event.data) + BOT.event_manager(event_out) + +def register(bot): + global BOT + BOT=bot + bot.prefix=">" + bot.event_manager.on("privmsg",on_privmsg) diff --git a/plugins/search_plugin.py b/plugins/search_plugin.py new file mode 100644 index 0000000..591fa47 --- /dev/null +++ b/plugins/search_plugin.py @@ -0,0 +1,28 @@ +from bot import IRCLine +BOT = None + +def respond(event,msg): + if event.target.startswith("#"): + prefix = event.hostmask.nick+": " + target = event.target + else: + prefix = "" + target = event.hostmask.nick + BOT.socket.send(IRCLine("PRIVMSG",target,":"+msg).line) + +import search + +def _search(event): + query = " ".join(event.parts) + results = search.search(query) + if len(results)==0: + respond(event,"No results found for \"{}\"".format(query)) + return + respond(event,"Top result: {result[0]} - {result[1]}".format(result=results[0])) + +def register(bot): + global BOT + BOT=bot + bot.event_manager.on("command_search",_search) + bot.event_manager.on("command_help",_search) + bot.event_manager.on("command_wiki",_search) diff --git a/search.py b/search.py new file mode 100644 index 0000000..48e89df --- /dev/null +++ b/search.py @@ -0,0 +1,12 @@ +from whoosh import index, qparser + +_index = index.open_dir("_index") +parser = qparser.QueryParser("text",_index.schema) + +def search(q): + q = parser.parse(q) + out = [] + with _index.searcher() as s: + for result in s.search(q,limit=None): + out.append((result["title"],result["url"])) + return out