Add auto-retry on 429 when listing emojis for export
This commit is contained in:
parent
75f3e2220e
commit
39a94276d5
14
export.py
14
export.py
|
@ -9,6 +9,7 @@ import asyncio
|
||||||
import logging
|
import logging
|
||||||
import lxml.html
|
import lxml.html
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
import re
|
import re
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
|
|
||||||
|
@ -120,12 +121,17 @@ async def _determine_all_emoji_urls(session: aiohttp.ClientSession, base_url: st
|
||||||
'count': 100
|
'count': 100
|
||||||
}
|
}
|
||||||
|
|
||||||
response = await session.post(base_url + EMOJI_API, data=data)
|
while True:
|
||||||
|
response = await session.post(base_url + EMOJI_API, data=data)
|
||||||
|
|
||||||
logger.info(f"loaded {response.real_url} (page {page})")
|
logger.info(f"loaded {response.real_url} (page {page})")
|
||||||
|
if response.status == 200:
|
||||||
|
break
|
||||||
|
if response.status != 429:
|
||||||
|
raise Exception(f"Failed to load emoji from {response.request_info.real_url} (status {response.status})")
|
||||||
|
|
||||||
if response.status != 200:
|
logger.warning('Too many requests! Retrying in 5 seconds')
|
||||||
raise Exception(f"Failed to load emoji from {response.request_info.real_url} (status {response.status})")
|
time.sleep(5)
|
||||||
|
|
||||||
json = await response.json()
|
json = await response.json()
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue