envs
/
twtxt
mirror of https://git.envs.net/envs/twtxt.git
1
0
Fork 0
Code Releases Activity

Adding option to specify minimal interval between checks (closes #100) 

The option is called 'timeline_update_interval', defaults to 10.
The behaviour is as follows: If cache is enabled and was updated
in the last 'timeline_update_interval' seconds, skip any network
requests and load the timline completely from the cache, otherwise
proceed with network requests and add new content to the cache, if
necessary. This only works if the cache is enabled.

An overwrite switch is also added, called: --force-update
When specified network requests will be made to check if
cache content is still up-to-date, but the cache will still
be used, when possible. Use --no-cache if you want to disable
the caching.
This commit is contained in:
buckket 2016-03-30 21:26:03 +02:00
parent a62df25e44
commit 99a17dd236
7 changed files with 131 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.example
View File

@ -10,6 +10,7 @@ character_limit = 140
character_warning = 140
disclose_identity = False
limit_timeline = 20
timeline_update_interval = 10
timeout = 5.0
sorting = descending
use_abs_time = False

73
docs/user/configuration.rst
View File

@ -21,6 +21,7 @@ Here’s an example ``conf`` file, showing every currently supported option:
character_limit = 140
character_warning = 140
limit_timeline = 20
timeline_update_interval = 10
timeout = 5.0
sorting = descending
pre_tweet_hook = "scp buckket@example.org:~/public_html/twtxt.txt {twtfile}"
@ -35,41 +36,43 @@ Here’s an example ``conf`` file, showing every currently supported option:
[twtxt]
-------
+-------------------+-------+------------+---------------------------------------------------+
| Option: | Type: | Default: | Help: |
+===================+=======+============+===================================================+
| nick | TEXT | | your nick, will be displayed in your timeline |
+-------------------+-------+------------+---------------------------------------------------+
| twtfile | PATH | | path to your local twtxt file |
+-------------------+-------+------------+---------------------------------------------------+
| twturl | TEXT | | URL to your public twtxt file |
+-------------------+-------+------------+---------------------------------------------------+
| check_following | BOOL | True | try to resolve URLs when listing followings |
+-------------------+-------+------------+---------------------------------------------------+
| use_pager | BOOL | False | use a pager (less) to display your timeline |
+-------------------+-------+------------+---------------------------------------------------+
| use_cache | BOOL | True | cache remote twtxt files locally |
+-------------------+-------+------------+---------------------------------------------------+
| porcelain | BOOL | False | style output in an easy-to-parse format |
+-------------------+-------+------------+---------------------------------------------------+
| disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent |
+-------------------+-------+------------+---------------------------------------------------+
| character_limit | INT | None | shorten incoming tweets with more characters |
+-------------------+-------+------------+---------------------------------------------------+
| character_warning | INT | None | warn when composed tweet has more characters |
+-------------------+-------+------------+---------------------------------------------------+
| limit_timeline | INT | 20 | limit amount of tweets shown in your timeline |
+-------------------+-------+------------+---------------------------------------------------+
| timeout | FLOAT | 5.0 | maximal time a http request is allowed to take |
+-------------------+-------+------------+---------------------------------------------------+
| sorting | TEXT | descending | sort timeline either descending or ascending |
+-------------------+-------+------------+---------------------------------------------------+
| use_abs_time | BOOL | False | use absolute datetimes in your timeline |
+-------------------+-------+------------+---------------------------------------------------+
| pre_tweet_hook | TEXT | | command to be executed before tweeting |
+-------------------+-------+------------+---------------------------------------------------+
| post_tweet_hook | TEXT | | command to be executed after tweeting |
+-------------------+-------+------------+---------------------------------------------------+
+---------------------------+-------+------------+---------------------------------------------------+
| Option: | Type: | Default: | Help: |
+===========================+=======+============+===================================================+
| nick | TEXT | | your nick, will be displayed in your timeline |
+---------------------------+-------+------------+---------------------------------------------------+
| twtfile | PATH | | path to your local twtxt file |
+---------------------------+-------+------------+---------------------------------------------------+
| twturl | TEXT | | URL to your public twtxt file |
+---------------------------+-------+------------+---------------------------------------------------+
| check_following | BOOL | True | try to resolve URLs when listing followings |
+---------------------------+-------+------------+---------------------------------------------------+
| use_pager | BOOL | False | use a pager (less) to display your timeline |
+---------------------------+-------+------------+---------------------------------------------------+
| use_cache | BOOL | True | cache remote twtxt files locally |
+---------------------------+-------+------------+---------------------------------------------------+
| porcelain | BOOL | False | style output in an easy-to-parse format |
+---------------------------+-------+------------+---------------------------------------------------+
| disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent |
+---------------------------+-------+------------+---------------------------------------------------+
| character_limit | INT | None | shorten incoming tweets with more characters |
+---------------------------+-------+------------+---------------------------------------------------+
| character_warning | INT | None | warn when composed tweet has more characters |
+---------------------------+-------+------------+---------------------------------------------------+
| limit_timeline | INT | 20 | limit amount of tweets shown in your timeline |
+---------------------------+-------+------------+---------------------------------------------------+
| timeline_update_interval | INT | 10 | time in seconds cache is considered up-to-date |
+---------------------------+-------+------------+---------------------------------------------------+
| timeout | FLOAT | 5.0 | maximal time a http request is allowed to take |
+---------------------------+-------+------------+---------------------------------------------------+
| sorting | TEXT | descending | sort timeline either descending or ascending |
+---------------------------+-------+------------+---------------------------------------------------+
| use_abs_time | BOOL | False | use absolute datetimes in your timeline |
+---------------------------+-------+------------+---------------------------------------------------+
| pre_tweet_hook | TEXT | | command to be executed before tweeting |
+---------------------------+-------+------------+---------------------------------------------------+
| post_tweet_hook | TEXT | | command to be executed after tweeting |
+---------------------------+-------+------------+---------------------------------------------------+
``pre_tweet_hook`` and ``post_tweet_hook`` are very useful if you want to push your twtxt file to a remote (web) server. Check the example above tho see how it’s used with ``scp``.

5
tests/test_config.py
View File

@ -23,6 +23,7 @@ def config_dir(tmpdir_factory):
cfg.set("twtxt", "character_warning", "150")
cfg.set("twtxt", "disclose_identity", "True")
cfg.set("twtxt", "limit_timeline", "50")
cfg.set("twtxt", "timeline_update_interval", "20")
cfg.set("twtxt", "timeout", "1.0")
cfg.set("twtxt", "sorting", "ascending")
cfg.set("twtxt", "post_tweet_hook", "echo {twtfile")
@ -71,6 +72,7 @@ def test_defaults():
assert empty_conf.character_warning is None
assert empty_conf.disclose_identity is False
assert empty_conf.limit_timeline == 20
assert empty_conf.timeline_update_interval == 10
assert empty_conf.timeout == 5.0
assert empty_conf.sorting == "descending"
assert empty_conf.post_tweet_hook is None
@ -89,6 +91,7 @@ def check_cfg(cfg):
assert cfg.character_warning == 150
assert cfg.disclose_identity is True
assert cfg.limit_timeline == 50
assert cfg.timeline_update_interval == 20
assert cfg.timeout == 1.0
assert cfg.sorting == "ascending"
assert cfg.post_tweet_hook == "echo {twtfile"
@ -175,6 +178,7 @@ def test_build_default_map():
"sorting": empty_conf.sorting,
"porcelain": empty_conf.porcelain,
"twtfile": empty_conf.twtfile,
"update_interval": empty_conf.timeline_update_interval,
},
"view": {
"pager": empty_conf.use_pager,
@ -183,6 +187,7 @@ def test_build_default_map():
"timeout": empty_conf.timeout,
"sorting": empty_conf.sorting,
"porcelain": empty_conf.porcelain,
"update_interval": empty_conf.timeline_update_interval,
}
}

52
twtxt/cache.py
View File

@ -11,27 +11,28 @@
import logging
import os
import shelve
from time import time as timestamp
import click
from click import get_app_dir
logger = logging.getLogger(__name__)
class Cache:
cache_dir = click.get_app_dir("twtxt")
cache_dir = get_app_dir("twtxt")
cache_name = "cache"
def __init__(self, cache_file, cache):
def __init__(self, cache_file, cache, update_interval):
"""Initializes new :class:`Cache` object.
:param cache_file: full path to the loaded cache file.
:param cache: a Shelve object, with cache loaded.
:param str cache_file: full path to the loaded cache file.
:param ~shelve.Shelve cache: a Shelve object, with cache loaded.
:param int update_interval: number of seconds the cache is considered to be
up-to-date without calling any external resources.
"""
self.cache_file = cache_file
self.cache = cache
self.cache["last_update"] = self.cache.get("last_update") or 0
self.update_interval = update_interval
def __enter__(self):
return self
@ -40,20 +41,41 @@ class Cache:
return self.close()
@classmethod
def from_file(cls, file):
def from_file(cls, file, *args, **kwargs):
"""Try loading given cache file."""
try:
cache = shelve.open(file)
return cls(file, cache)
return cls(file, cache, *args, **kwargs)
except OSError as e:
logger.debug("Loading {0} failed".format(file))
raise e
@classmethod
def discover(cls):
def discover(cls, *args, **kwargs):
"""Make a guess about the cache file location an try loading it."""
file = os.path.join(Cache.cache_dir, Cache.cache_name)
return cls.from_file(file)
return cls.from_file(file, *args, **kwargs)
@property
def last_updated(self):
"""Returns *NIX timestamp of last update of the cache."""
try:
return self.cache["last_update"]
except KeyError:
return 0
@property
def is_valid(self):
"""Checks if the cache is considered to be up-to-date."""
if timestamp() - self.last_updated <= self.update_interval:
return True
else:
return False
def mark_updated(self):
"""Mark cache as updated at current *NIX timestamp"""
if not self.is_valid:
self.cache["last_update"] = timestamp()
def is_cached(self, url):
"""Checks if specified URL is cached."""
@ -62,14 +84,6 @@ class Cache:
except TypeError:
return False
def last_updated(self):
"""Returns *NIX timestamp of last update of the cache."""
return self.cache["last_update"]
def mark_updated(self):
"""Mark Cache as updated at current *NIX timestamp"""
self.cache["last_update"] = timestamp()
def last_modified(self, url):
"""Returns saved 'Last-Modified' header, if available."""
try:

77
twtxt/cli.py
View File

@ -12,7 +12,7 @@ import logging
import os
import sys
import textwrap
from time import time as timestamp
from itertools import chain
import click
@ -99,21 +99,26 @@ def tweet(ctx, created_at, twtfile, text):
@click.option("--twtfile", "-f",
type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
help="Location of your twtxt file. (Default: twtxt.txt")
@click.option("--ascending", "sorting", flag_value="ascending",
@click.option("--ascending", "sorting",
flag_value="ascending",
help="Sort timeline in ascending order.")
@click.option("--descending", "sorting", flag_value="descending",
@click.option("--descending", "sorting",
flag_value="descending",
help="Sort timeline in descending order. (Default)")
@click.option("--timeout", type=click.FLOAT,
@click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True,
@click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)")
@click.option("--source", "-s",
help="Only show feed of the given source. (Can be nick or URL)")
@click.option("--cache/--no-cache",
is_flag=True,
help="Cache remote twtxt files locally. (Default: True)")
@click.option("--force-update", is_flag=True,
help="Force update even when timeline is called multiple times in a short period of time")
@click.option("--force-update",
is_flag=True,
help="Force update even if cache is up-to-date. (Default: False)")
@click.pass_context
def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update):
"""Retrieve your personal timeline."""
@ -126,17 +131,22 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
else:
sources = ctx.obj["conf"].following
tweets = []
with Cache.discover() as cache:
force_update = force_update or timestamp() - cache.last_updated() >= ctx.obj["conf"].timeline_update_interval
if force_update:
tweets = get_remote_tweets(sources, limit, timeout)
if cache:
try:
with Cache.discover(update_interval=ctx.obj["conf"].timeline_update_interval) as cache:
force_update = force_update or not cache.is_valid
if force_update:
tweets = get_remote_tweets(sources, limit, timeout, cache)
else:
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(
cache.update_interval))
# Behold, almighty list comprehensions! (I might have gone overboard here…)
tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))
except OSError as e:
logger.debug(e)
tweets = get_remote_tweets(sources, limit, timeout)
else:
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(ctx.obj["conf"].timeline_update_interval))
with Cache.discover() as cache:
for source in sources:
tweets += cache.get_tweets(source.url)
tweets = get_remote_tweets(sources, limit, timeout)
if twtfile and not source:
source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile)
@ -160,17 +170,24 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
@click.option("--limit", "-l",
type=click.INT,
help="Limit total number of shown tweets. (Default: 20)")
@click.option("--ascending", "sorting", flag_value="ascending",
@click.option("--ascending", "sorting",
flag_value="ascending",
help="Sort timeline in ascending order.")
@click.option("--descending", "sorting", flag_value="descending",
@click.option("--descending", "sorting",
flag_value="descending",
help="Sort timeline in descending order. (Default)")
@click.option("--timeout", type=click.FLOAT,
@click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True,
@click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)")
@click.option("--cache/--no-cache",
is_flag=True,
help="Cache remote twtxt files locally. (Default: True)")
@click.option("--force-update",
is_flag=True,
help="Force update even if cache is up-to-date. (Default: False)")
@click.argument("source")
@click.pass_context
def view(ctx, **kwargs):
@ -182,9 +199,11 @@ def view(ctx, **kwargs):
@click.option("--check/--no-check",
is_flag=True,
help="Check if source URL is valid and readable. (Default: True)")
@click.option("--timeout", type=click.FLOAT,
@click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True,
@click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)")
@click.pass_context
def following(ctx, check, timeout, porcelain):
@ -204,7 +223,8 @@ def following(ctx, check, timeout, porcelain):
@cli.command()
@click.argument("nick")
@click.argument("url")
@click.option("--force", "-f", flag_value=True,
@click.option("--force", "-f",
flag_value=True,
help="Force adding and overwriting nick")
@click.pass_context
def follow(ctx, nick, url, force):
@ -283,7 +303,8 @@ def quickstart():
twtfile = os.path.expanduser(twtfile)
overwrite_check(twtfile)
disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared", default=False)
disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared",
default=False)
click.echo()
add_news = click.confirm("➤ Do you want to follow the twtxt news feed?", default=True)
@ -303,9 +324,11 @@ def quickstart():
@cli.command()
@click.argument("key", required=False, callback=validate_config_key)
@click.argument("value", required=False)
@click.option("--remove", flag_value=True,
@click.option("--remove",
flag_value=True,
help="Remove given item")
@click.option("--edit", "-e", flag_value=True,
@click.option("--edit", "-e",
flag_value=True,
help="Open config file in editor")
@click.pass_context
def config(ctx, key, value, remove, edit):

3
twtxt/config.py
View File

@ -163,7 +163,7 @@ class Config:
@property
def timeline_update_interval(self):
return self.cfg.getint("twtxt", "timeline_update_interval", fallback = 10)
return self.cfg.getint("twtxt", "timeline_update_interval", fallback=10)
@property
def use_abs_time(self):
@ -245,6 +245,7 @@ class Config:
"timeout": self.timeout,
"sorting": self.sorting,
"porcelain": self.porcelain,
"update_interval": self.timeline_update_interval,
}
}
return default_map

12
twtxt/twhttp.py
View File

@ -107,7 +107,7 @@ def process_sources_for_file(client, sources, limit, cache=None):
return sorted(g_tweets, reverse=True)[:limit]
def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True):
def get_remote_tweets(sources, limit=None, timeout=5.0, cache=None):
conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True)
headers = generate_user_agent()
with aiohttp.ClientSession(connector=conn, headers=headers) as client:
@ -116,15 +116,7 @@ def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True):
def start_loop(client, sources, limit, cache=None):
return loop.run_until_complete(process_sources_for_file(client, sources, limit, cache))
if use_cache:
try:
with Cache.discover() as cache:
tweets = start_loop(client, sources, limit, cache)
except OSError as e:
logger.debug(e)
tweets = start_loop(client, sources, limit)
else:
tweets = start_loop(client, sources, limit)
tweets = start_loop(client, sources, limit, cache)
return tweets