mirror of
https://git.envs.net/envs/twtxt.git
synced 2024-09-07 22:53:15 +00:00
Adding option to specify minimal interval between checks (closes #100)
The option is called 'timeline_update_interval', defaults to 10. The behaviour is as follows: If cache is enabled and was updated in the last 'timeline_update_interval' seconds, skip any network requests and load the timline completely from the cache, otherwise proceed with network requests and add new content to the cache, if necessary. This only works if the cache is enabled. An overwrite switch is also added, called: --force-update When specified network requests will be made to check if cache content is still up-to-date, but the cache will still be used, when possible. Use --no-cache if you want to disable the caching.
This commit is contained in:
parent
a62df25e44
commit
99a17dd236
@ -10,6 +10,7 @@ character_limit = 140
|
||||
character_warning = 140
|
||||
disclose_identity = False
|
||||
limit_timeline = 20
|
||||
timeline_update_interval = 10
|
||||
timeout = 5.0
|
||||
sorting = descending
|
||||
use_abs_time = False
|
||||
|
@ -21,6 +21,7 @@ Here’s an example ``conf`` file, showing every currently supported option:
|
||||
character_limit = 140
|
||||
character_warning = 140
|
||||
limit_timeline = 20
|
||||
timeline_update_interval = 10
|
||||
timeout = 5.0
|
||||
sorting = descending
|
||||
pre_tweet_hook = "scp buckket@example.org:~/public_html/twtxt.txt {twtfile}"
|
||||
@ -35,41 +36,43 @@ Here’s an example ``conf`` file, showing every currently supported option:
|
||||
[twtxt]
|
||||
-------
|
||||
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| Option: | Type: | Default: | Help: |
|
||||
+===================+=======+============+===================================================+
|
||||
| nick | TEXT | | your nick, will be displayed in your timeline |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| twtfile | PATH | | path to your local twtxt file |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| twturl | TEXT | | URL to your public twtxt file |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| check_following | BOOL | True | try to resolve URLs when listing followings |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| use_pager | BOOL | False | use a pager (less) to display your timeline |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| use_cache | BOOL | True | cache remote twtxt files locally |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| porcelain | BOOL | False | style output in an easy-to-parse format |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| character_limit | INT | None | shorten incoming tweets with more characters |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| character_warning | INT | None | warn when composed tweet has more characters |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| limit_timeline | INT | 20 | limit amount of tweets shown in your timeline |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| timeout | FLOAT | 5.0 | maximal time a http request is allowed to take |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| sorting | TEXT | descending | sort timeline either descending or ascending |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| use_abs_time | BOOL | False | use absolute datetimes in your timeline |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| pre_tweet_hook | TEXT | | command to be executed before tweeting |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
| post_tweet_hook | TEXT | | command to be executed after tweeting |
|
||||
+-------------------+-------+------------+---------------------------------------------------+
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| Option: | Type: | Default: | Help: |
|
||||
+===========================+=======+============+===================================================+
|
||||
| nick | TEXT | | your nick, will be displayed in your timeline |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| twtfile | PATH | | path to your local twtxt file |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| twturl | TEXT | | URL to your public twtxt file |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| check_following | BOOL | True | try to resolve URLs when listing followings |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| use_pager | BOOL | False | use a pager (less) to display your timeline |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| use_cache | BOOL | True | cache remote twtxt files locally |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| porcelain | BOOL | False | style output in an easy-to-parse format |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| character_limit | INT | None | shorten incoming tweets with more characters |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| character_warning | INT | None | warn when composed tweet has more characters |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| limit_timeline | INT | 20 | limit amount of tweets shown in your timeline |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| timeline_update_interval | INT | 10 | time in seconds cache is considered up-to-date |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| timeout | FLOAT | 5.0 | maximal time a http request is allowed to take |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| sorting | TEXT | descending | sort timeline either descending or ascending |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| use_abs_time | BOOL | False | use absolute datetimes in your timeline |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| pre_tweet_hook | TEXT | | command to be executed before tweeting |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
| post_tweet_hook | TEXT | | command to be executed after tweeting |
|
||||
+---------------------------+-------+------------+---------------------------------------------------+
|
||||
|
||||
``pre_tweet_hook`` and ``post_tweet_hook`` are very useful if you want to push your twtxt file to a remote (web) server. Check the example above tho see how it’s used with ``scp``.
|
||||
|
||||
|
@ -23,6 +23,7 @@ def config_dir(tmpdir_factory):
|
||||
cfg.set("twtxt", "character_warning", "150")
|
||||
cfg.set("twtxt", "disclose_identity", "True")
|
||||
cfg.set("twtxt", "limit_timeline", "50")
|
||||
cfg.set("twtxt", "timeline_update_interval", "20")
|
||||
cfg.set("twtxt", "timeout", "1.0")
|
||||
cfg.set("twtxt", "sorting", "ascending")
|
||||
cfg.set("twtxt", "post_tweet_hook", "echo {twtfile")
|
||||
@ -71,6 +72,7 @@ def test_defaults():
|
||||
assert empty_conf.character_warning is None
|
||||
assert empty_conf.disclose_identity is False
|
||||
assert empty_conf.limit_timeline == 20
|
||||
assert empty_conf.timeline_update_interval == 10
|
||||
assert empty_conf.timeout == 5.0
|
||||
assert empty_conf.sorting == "descending"
|
||||
assert empty_conf.post_tweet_hook is None
|
||||
@ -89,6 +91,7 @@ def check_cfg(cfg):
|
||||
assert cfg.character_warning == 150
|
||||
assert cfg.disclose_identity is True
|
||||
assert cfg.limit_timeline == 50
|
||||
assert cfg.timeline_update_interval == 20
|
||||
assert cfg.timeout == 1.0
|
||||
assert cfg.sorting == "ascending"
|
||||
assert cfg.post_tweet_hook == "echo {twtfile"
|
||||
@ -175,6 +178,7 @@ def test_build_default_map():
|
||||
"sorting": empty_conf.sorting,
|
||||
"porcelain": empty_conf.porcelain,
|
||||
"twtfile": empty_conf.twtfile,
|
||||
"update_interval": empty_conf.timeline_update_interval,
|
||||
},
|
||||
"view": {
|
||||
"pager": empty_conf.use_pager,
|
||||
@ -183,6 +187,7 @@ def test_build_default_map():
|
||||
"timeout": empty_conf.timeout,
|
||||
"sorting": empty_conf.sorting,
|
||||
"porcelain": empty_conf.porcelain,
|
||||
"update_interval": empty_conf.timeline_update_interval,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -11,27 +11,28 @@
|
||||
import logging
|
||||
import os
|
||||
import shelve
|
||||
|
||||
from time import time as timestamp
|
||||
|
||||
import click
|
||||
from click import get_app_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Cache:
|
||||
cache_dir = click.get_app_dir("twtxt")
|
||||
cache_dir = get_app_dir("twtxt")
|
||||
cache_name = "cache"
|
||||
|
||||
def __init__(self, cache_file, cache):
|
||||
def __init__(self, cache_file, cache, update_interval):
|
||||
"""Initializes new :class:`Cache` object.
|
||||
|
||||
:param cache_file: full path to the loaded cache file.
|
||||
:param cache: a Shelve object, with cache loaded.
|
||||
:param str cache_file: full path to the loaded cache file.
|
||||
:param ~shelve.Shelve cache: a Shelve object, with cache loaded.
|
||||
:param int update_interval: number of seconds the cache is considered to be
|
||||
up-to-date without calling any external resources.
|
||||
"""
|
||||
self.cache_file = cache_file
|
||||
self.cache = cache
|
||||
self.cache["last_update"] = self.cache.get("last_update") or 0
|
||||
self.update_interval = update_interval
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
@ -40,20 +41,41 @@ class Cache:
|
||||
return self.close()
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, file):
|
||||
def from_file(cls, file, *args, **kwargs):
|
||||
"""Try loading given cache file."""
|
||||
try:
|
||||
cache = shelve.open(file)
|
||||
return cls(file, cache)
|
||||
return cls(file, cache, *args, **kwargs)
|
||||
except OSError as e:
|
||||
logger.debug("Loading {0} failed".format(file))
|
||||
raise e
|
||||
|
||||
@classmethod
|
||||
def discover(cls):
|
||||
def discover(cls, *args, **kwargs):
|
||||
"""Make a guess about the cache file location an try loading it."""
|
||||
file = os.path.join(Cache.cache_dir, Cache.cache_name)
|
||||
return cls.from_file(file)
|
||||
return cls.from_file(file, *args, **kwargs)
|
||||
|
||||
@property
|
||||
def last_updated(self):
|
||||
"""Returns *NIX timestamp of last update of the cache."""
|
||||
try:
|
||||
return self.cache["last_update"]
|
||||
except KeyError:
|
||||
return 0
|
||||
|
||||
@property
|
||||
def is_valid(self):
|
||||
"""Checks if the cache is considered to be up-to-date."""
|
||||
if timestamp() - self.last_updated <= self.update_interval:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def mark_updated(self):
|
||||
"""Mark cache as updated at current *NIX timestamp"""
|
||||
if not self.is_valid:
|
||||
self.cache["last_update"] = timestamp()
|
||||
|
||||
def is_cached(self, url):
|
||||
"""Checks if specified URL is cached."""
|
||||
@ -62,14 +84,6 @@ class Cache:
|
||||
except TypeError:
|
||||
return False
|
||||
|
||||
def last_updated(self):
|
||||
"""Returns *NIX timestamp of last update of the cache."""
|
||||
return self.cache["last_update"]
|
||||
|
||||
def mark_updated(self):
|
||||
"""Mark Cache as updated at current *NIX timestamp"""
|
||||
self.cache["last_update"] = timestamp()
|
||||
|
||||
def last_modified(self, url):
|
||||
"""Returns saved 'Last-Modified' header, if available."""
|
||||
try:
|
||||
|
77
twtxt/cli.py
77
twtxt/cli.py
@ -12,7 +12,7 @@ import logging
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
from time import time as timestamp
|
||||
from itertools import chain
|
||||
|
||||
import click
|
||||
|
||||
@ -99,21 +99,26 @@ def tweet(ctx, created_at, twtfile, text):
|
||||
@click.option("--twtfile", "-f",
|
||||
type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
|
||||
help="Location of your twtxt file. (Default: twtxt.txt")
|
||||
@click.option("--ascending", "sorting", flag_value="ascending",
|
||||
@click.option("--ascending", "sorting",
|
||||
flag_value="ascending",
|
||||
help="Sort timeline in ascending order.")
|
||||
@click.option("--descending", "sorting", flag_value="descending",
|
||||
@click.option("--descending", "sorting",
|
||||
flag_value="descending",
|
||||
help="Sort timeline in descending order. (Default)")
|
||||
@click.option("--timeout", type=click.FLOAT,
|
||||
@click.option("--timeout",
|
||||
type=click.FLOAT,
|
||||
help="Maximum time requests are allowed to take. (Default: 5.0)")
|
||||
@click.option("--porcelain", is_flag=True,
|
||||
@click.option("--porcelain",
|
||||
is_flag=True,
|
||||
help="Style output in an easy-to-parse format. (Default: False)")
|
||||
@click.option("--source", "-s",
|
||||
help="Only show feed of the given source. (Can be nick or URL)")
|
||||
@click.option("--cache/--no-cache",
|
||||
is_flag=True,
|
||||
help="Cache remote twtxt files locally. (Default: True)")
|
||||
@click.option("--force-update", is_flag=True,
|
||||
help="Force update even when timeline is called multiple times in a short period of time")
|
||||
@click.option("--force-update",
|
||||
is_flag=True,
|
||||
help="Force update even if cache is up-to-date. (Default: False)")
|
||||
@click.pass_context
|
||||
def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update):
|
||||
"""Retrieve your personal timeline."""
|
||||
@ -126,17 +131,22 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
|
||||
else:
|
||||
sources = ctx.obj["conf"].following
|
||||
|
||||
tweets = []
|
||||
with Cache.discover() as cache:
|
||||
force_update = force_update or timestamp() - cache.last_updated() >= ctx.obj["conf"].timeline_update_interval
|
||||
|
||||
if force_update:
|
||||
tweets = get_remote_tweets(sources, limit, timeout)
|
||||
if cache:
|
||||
try:
|
||||
with Cache.discover(update_interval=ctx.obj["conf"].timeline_update_interval) as cache:
|
||||
force_update = force_update or not cache.is_valid
|
||||
if force_update:
|
||||
tweets = get_remote_tweets(sources, limit, timeout, cache)
|
||||
else:
|
||||
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(
|
||||
cache.update_interval))
|
||||
# Behold, almighty list comprehensions! (I might have gone overboard here…)
|
||||
tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))
|
||||
except OSError as e:
|
||||
logger.debug(e)
|
||||
tweets = get_remote_tweets(sources, limit, timeout)
|
||||
else:
|
||||
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(ctx.obj["conf"].timeline_update_interval))
|
||||
with Cache.discover() as cache:
|
||||
for source in sources:
|
||||
tweets += cache.get_tweets(source.url)
|
||||
tweets = get_remote_tweets(sources, limit, timeout)
|
||||
|
||||
if twtfile and not source:
|
||||
source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile)
|
||||
@ -160,17 +170,24 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
|
||||
@click.option("--limit", "-l",
|
||||
type=click.INT,
|
||||
help="Limit total number of shown tweets. (Default: 20)")
|
||||
@click.option("--ascending", "sorting", flag_value="ascending",
|
||||
@click.option("--ascending", "sorting",
|
||||
flag_value="ascending",
|
||||
help="Sort timeline in ascending order.")
|
||||
@click.option("--descending", "sorting", flag_value="descending",
|
||||
@click.option("--descending", "sorting",
|
||||
flag_value="descending",
|
||||
help="Sort timeline in descending order. (Default)")
|
||||
@click.option("--timeout", type=click.FLOAT,
|
||||
@click.option("--timeout",
|
||||
type=click.FLOAT,
|
||||
help="Maximum time requests are allowed to take. (Default: 5.0)")
|
||||
@click.option("--porcelain", is_flag=True,
|
||||
@click.option("--porcelain",
|
||||
is_flag=True,
|
||||
help="Style output in an easy-to-parse format. (Default: False)")
|
||||
@click.option("--cache/--no-cache",
|
||||
is_flag=True,
|
||||
help="Cache remote twtxt files locally. (Default: True)")
|
||||
@click.option("--force-update",
|
||||
is_flag=True,
|
||||
help="Force update even if cache is up-to-date. (Default: False)")
|
||||
@click.argument("source")
|
||||
@click.pass_context
|
||||
def view(ctx, **kwargs):
|
||||
@ -182,9 +199,11 @@ def view(ctx, **kwargs):
|
||||
@click.option("--check/--no-check",
|
||||
is_flag=True,
|
||||
help="Check if source URL is valid and readable. (Default: True)")
|
||||
@click.option("--timeout", type=click.FLOAT,
|
||||
@click.option("--timeout",
|
||||
type=click.FLOAT,
|
||||
help="Maximum time requests are allowed to take. (Default: 5.0)")
|
||||
@click.option("--porcelain", is_flag=True,
|
||||
@click.option("--porcelain",
|
||||
is_flag=True,
|
||||
help="Style output in an easy-to-parse format. (Default: False)")
|
||||
@click.pass_context
|
||||
def following(ctx, check, timeout, porcelain):
|
||||
@ -204,7 +223,8 @@ def following(ctx, check, timeout, porcelain):
|
||||
@cli.command()
|
||||
@click.argument("nick")
|
||||
@click.argument("url")
|
||||
@click.option("--force", "-f", flag_value=True,
|
||||
@click.option("--force", "-f",
|
||||
flag_value=True,
|
||||
help="Force adding and overwriting nick")
|
||||
@click.pass_context
|
||||
def follow(ctx, nick, url, force):
|
||||
@ -283,7 +303,8 @@ def quickstart():
|
||||
twtfile = os.path.expanduser(twtfile)
|
||||
overwrite_check(twtfile)
|
||||
|
||||
disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared", default=False)
|
||||
disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared",
|
||||
default=False)
|
||||
|
||||
click.echo()
|
||||
add_news = click.confirm("➤ Do you want to follow the twtxt news feed?", default=True)
|
||||
@ -303,9 +324,11 @@ def quickstart():
|
||||
@cli.command()
|
||||
@click.argument("key", required=False, callback=validate_config_key)
|
||||
@click.argument("value", required=False)
|
||||
@click.option("--remove", flag_value=True,
|
||||
@click.option("--remove",
|
||||
flag_value=True,
|
||||
help="Remove given item")
|
||||
@click.option("--edit", "-e", flag_value=True,
|
||||
@click.option("--edit", "-e",
|
||||
flag_value=True,
|
||||
help="Open config file in editor")
|
||||
@click.pass_context
|
||||
def config(ctx, key, value, remove, edit):
|
||||
|
@ -163,7 +163,7 @@ class Config:
|
||||
|
||||
@property
|
||||
def timeline_update_interval(self):
|
||||
return self.cfg.getint("twtxt", "timeline_update_interval", fallback = 10)
|
||||
return self.cfg.getint("twtxt", "timeline_update_interval", fallback=10)
|
||||
|
||||
@property
|
||||
def use_abs_time(self):
|
||||
@ -245,6 +245,7 @@ class Config:
|
||||
"timeout": self.timeout,
|
||||
"sorting": self.sorting,
|
||||
"porcelain": self.porcelain,
|
||||
"update_interval": self.timeline_update_interval,
|
||||
}
|
||||
}
|
||||
return default_map
|
||||
|
@ -107,7 +107,7 @@ def process_sources_for_file(client, sources, limit, cache=None):
|
||||
return sorted(g_tweets, reverse=True)[:limit]
|
||||
|
||||
|
||||
def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True):
|
||||
def get_remote_tweets(sources, limit=None, timeout=5.0, cache=None):
|
||||
conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True)
|
||||
headers = generate_user_agent()
|
||||
with aiohttp.ClientSession(connector=conn, headers=headers) as client:
|
||||
@ -116,15 +116,7 @@ def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True):
|
||||
def start_loop(client, sources, limit, cache=None):
|
||||
return loop.run_until_complete(process_sources_for_file(client, sources, limit, cache))
|
||||
|
||||
if use_cache:
|
||||
try:
|
||||
with Cache.discover() as cache:
|
||||
tweets = start_loop(client, sources, limit, cache)
|
||||
except OSError as e:
|
||||
logger.debug(e)
|
||||
tweets = start_loop(client, sources, limit)
|
||||
else:
|
||||
tweets = start_loop(client, sources, limit)
|
||||
tweets = start_loop(client, sources, limit, cache)
|
||||
|
||||
return tweets
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user