diff --git a/export.py b/export.py index e2409c0..8842b6a 100755 --- a/export.py +++ b/export.py @@ -9,6 +9,7 @@ import asyncio import logging import lxml.html import os +import time import re from collections import namedtuple @@ -120,12 +121,17 @@ async def _determine_all_emoji_urls(session: aiohttp.ClientSession, base_url: st 'count': 100 } - response = await session.post(base_url + EMOJI_API, data=data) + while True: + response = await session.post(base_url + EMOJI_API, data=data) - logger.info(f"loaded {response.real_url} (page {page})") + logger.info(f"loaded {response.real_url} (page {page})") + if response.status == 200: + break + if response.status != 429: + raise Exception(f"Failed to load emoji from {response.request_info.real_url} (status {response.status})") - if response.status != 200: - raise Exception(f"Failed to load emoji from {response.request_info.real_url} (status {response.status})") + logger.warning('Too many requests! Retrying in 5 seconds') + time.sleep(5) json = await response.json()