envs/twtxt
1
0
Fork 0
mirror of https://git.envs.net/envs/twtxt.git synced 2024-09-07 22:53:15 +00:00
Code Releases Activity

Adding option to specify minimal interval between checks (closes #100)

Browse Source 
The option is called 'timeline_update_interval', defaults to 10.
The behaviour is as follows: If cache is enabled and was updated
in the last 'timeline_update_interval' seconds, skip any network
requests and load the timline completely from the cache, otherwise
proceed with network requests and add new content to the cache, if
necessary. This only works if the cache is enabled.

An overwrite switch is also added, called: --force-update
When specified network requests will be made to check if
cache content is still up-to-date, but the cache will still
be used, when possible. Use --no-cache if you want to disable
the caching.
This commit is contained in:
buckket 2016-03-30 21:26:03 +02:00
parent a62df25e44
commit 99a17dd236
7 changed files with 131 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.example
View File

@ -10,6 +10,7 @@ character_limit = 140
character_warning = 140 character_warning = 140
disclose_identity = False disclose_identity = False
limit_timeline = 20 limit_timeline = 20
timeline_update_interval = 10
timeout = 5.0 timeout = 5.0
sorting = descending sorting = descending
use_abs_time = False use_abs_time = False

73
docs/user/configuration.rst
View File

@ -21,6 +21,7 @@ Here’s an example ``conf`` file, showing every currently supported option:
character_limit = 140 character_limit = 140
character_warning = 140 character_warning = 140
limit_timeline = 20 limit_timeline = 20
timeline_update_interval = 10
timeout = 5.0 timeout = 5.0
sorting = descending sorting = descending
pre_tweet_hook = "scp buckket@example.org:~/public_html/twtxt.txt {twtfile}" pre_tweet_hook = "scp buckket@example.org:~/public_html/twtxt.txt {twtfile}"
@ -35,41 +36,43 @@ Here’s an example ``conf`` file, showing every currently supported option:
[twtxt] [twtxt]
------- -------
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| Option: | Type: | Default: | Help: | | Option: | Type: | Default: | Help: |
+===================+=======+============+===================================================+ +===========================+=======+============+===================================================+
| nick | TEXT | | your nick, will be displayed in your timeline | | nick | TEXT | | your nick, will be displayed in your timeline |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| twtfile | PATH | | path to your local twtxt file | | twtfile | PATH | | path to your local twtxt file |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| twturl | TEXT | | URL to your public twtxt file | | twturl | TEXT | | URL to your public twtxt file |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| check_following | BOOL | True | try to resolve URLs when listing followings | | check_following | BOOL | True | try to resolve URLs when listing followings |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| use_pager | BOOL | False | use a pager (less) to display your timeline | | use_pager | BOOL | False | use a pager (less) to display your timeline |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| use_cache | BOOL | True | cache remote twtxt files locally | | use_cache | BOOL | True | cache remote twtxt files locally |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| porcelain | BOOL | False | style output in an easy-to-parse format | | porcelain | BOOL | False | style output in an easy-to-parse format |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent | | disclose_identity | BOOL | False | include nick and twturl in twtxt’s user-agent |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| character_limit | INT | None | shorten incoming tweets with more characters | | character_limit | INT | None | shorten incoming tweets with more characters |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| character_warning | INT | None | warn when composed tweet has more characters | | character_warning | INT | None | warn when composed tweet has more characters |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| limit_timeline | INT | 20 | limit amount of tweets shown in your timeline | | limit_timeline | INT | 20 | limit amount of tweets shown in your timeline |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| timeout | FLOAT | 5.0 | maximal time a http request is allowed to take | | timeline_update_interval | INT | 10 | time in seconds cache is considered up-to-date |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| sorting | TEXT | descending | sort timeline either descending or ascending | | timeout | FLOAT | 5.0 | maximal time a http request is allowed to take |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| use_abs_time | BOOL | False | use absolute datetimes in your timeline | | sorting | TEXT | descending | sort timeline either descending or ascending |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| pre_tweet_hook | TEXT | | command to be executed before tweeting | | use_abs_time | BOOL | False | use absolute datetimes in your timeline |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| post_tweet_hook | TEXT | | command to be executed after tweeting | | pre_tweet_hook | TEXT | | command to be executed before tweeting |
+-------------------+-------+------------+---------------------------------------------------+ +---------------------------+-------+------------+---------------------------------------------------+
| post_tweet_hook | TEXT | | command to be executed after tweeting |
+---------------------------+-------+------------+---------------------------------------------------+
``pre_tweet_hook`` and ``post_tweet_hook`` are very useful if you want to push your twtxt file to a remote (web) server. Check the example above tho see how it’s used with ``scp``. ``pre_tweet_hook`` and ``post_tweet_hook`` are very useful if you want to push your twtxt file to a remote (web) server. Check the example above tho see how it’s used with ``scp``.

5
tests/test_config.py
View File

@ -23,6 +23,7 @@ def config_dir(tmpdir_factory):
cfg.set("twtxt", "character_warning", "150") cfg.set("twtxt", "character_warning", "150")
cfg.set("twtxt", "disclose_identity", "True") cfg.set("twtxt", "disclose_identity", "True")
cfg.set("twtxt", "limit_timeline", "50") cfg.set("twtxt", "limit_timeline", "50")
cfg.set("twtxt", "timeline_update_interval", "20")
cfg.set("twtxt", "timeout", "1.0") cfg.set("twtxt", "timeout", "1.0")
cfg.set("twtxt", "sorting", "ascending") cfg.set("twtxt", "sorting", "ascending")
cfg.set("twtxt", "post_tweet_hook", "echo {twtfile") cfg.set("twtxt", "post_tweet_hook", "echo {twtfile")
@ -71,6 +72,7 @@ def test_defaults():
assert empty_conf.character_warning is None assert empty_conf.character_warning is None
assert empty_conf.disclose_identity is False assert empty_conf.disclose_identity is False
assert empty_conf.limit_timeline == 20 assert empty_conf.limit_timeline == 20
assert empty_conf.timeline_update_interval == 10
assert empty_conf.timeout == 5.0 assert empty_conf.timeout == 5.0
assert empty_conf.sorting == "descending" assert empty_conf.sorting == "descending"
assert empty_conf.post_tweet_hook is None assert empty_conf.post_tweet_hook is None
@ -89,6 +91,7 @@ def check_cfg(cfg):
assert cfg.character_warning == 150 assert cfg.character_warning == 150
assert cfg.disclose_identity is True assert cfg.disclose_identity is True
assert cfg.limit_timeline == 50 assert cfg.limit_timeline == 50
assert cfg.timeline_update_interval == 20
assert cfg.timeout == 1.0 assert cfg.timeout == 1.0
assert cfg.sorting == "ascending" assert cfg.sorting == "ascending"
assert cfg.post_tweet_hook == "echo {twtfile" assert cfg.post_tweet_hook == "echo {twtfile"
@ -175,6 +178,7 @@ def test_build_default_map():
"sorting": empty_conf.sorting, "sorting": empty_conf.sorting,
"porcelain": empty_conf.porcelain, "porcelain": empty_conf.porcelain,
"twtfile": empty_conf.twtfile, "twtfile": empty_conf.twtfile,
"update_interval": empty_conf.timeline_update_interval,
}, },
"view": { "view": {
"pager": empty_conf.use_pager, "pager": empty_conf.use_pager,
@ -183,6 +187,7 @@ def test_build_default_map():
"timeout": empty_conf.timeout, "timeout": empty_conf.timeout,
"sorting": empty_conf.sorting, "sorting": empty_conf.sorting,
"porcelain": empty_conf.porcelain, "porcelain": empty_conf.porcelain,
"update_interval": empty_conf.timeline_update_interval,
} }
} }

52
twtxt/cache.py
View File

@ -11,27 +11,28 @@
import logging import logging
import os import os
import shelve import shelve
from time import time as timestamp from time import time as timestamp
import click from click import get_app_dir
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Cache: class Cache:
cache_dir = click.get_app_dir("twtxt") cache_dir = get_app_dir("twtxt")
cache_name = "cache" cache_name = "cache"
def __init__(self, cache_file, cache): def __init__(self, cache_file, cache, update_interval):
"""Initializes new :class:`Cache` object. """Initializes new :class:`Cache` object.
:param cache_file: full path to the loaded cache file. :param str cache_file: full path to the loaded cache file.
:param cache: a Shelve object, with cache loaded. :param ~shelve.Shelve cache: a Shelve object, with cache loaded.
:param int update_interval: number of seconds the cache is considered to be
up-to-date without calling any external resources.
""" """
self.cache_file = cache_file self.cache_file = cache_file
self.cache = cache self.cache = cache
self.cache["last_update"] = self.cache.get("last_update") or 0 self.update_interval = update_interval
def __enter__(self): def __enter__(self):
return self return self
@ -40,20 +41,41 @@ class Cache:
return self.close() return self.close()
@classmethod @classmethod
def from_file(cls, file): def from_file(cls, file, *args, **kwargs):
"""Try loading given cache file.""" """Try loading given cache file."""
try: try:
cache = shelve.open(file) cache = shelve.open(file)
return cls(file, cache) return cls(file, cache, *args, **kwargs)
except OSError as e: except OSError as e:
logger.debug("Loading {0} failed".format(file)) logger.debug("Loading {0} failed".format(file))
raise e raise e
@classmethod @classmethod
def discover(cls): def discover(cls, *args, **kwargs):
"""Make a guess about the cache file location an try loading it.""" """Make a guess about the cache file location an try loading it."""
file = os.path.join(Cache.cache_dir, Cache.cache_name) file = os.path.join(Cache.cache_dir, Cache.cache_name)
return cls.from_file(file) return cls.from_file(file, *args, **kwargs)
@property
def last_updated(self):
"""Returns *NIX timestamp of last update of the cache."""
try:
return self.cache["last_update"]
except KeyError:
return 0
@property
def is_valid(self):
"""Checks if the cache is considered to be up-to-date."""
if timestamp() - self.last_updated <= self.update_interval:
return True
else:
return False
def mark_updated(self):
"""Mark cache as updated at current *NIX timestamp"""
if not self.is_valid:
self.cache["last_update"] = timestamp()
def is_cached(self, url): def is_cached(self, url):
"""Checks if specified URL is cached.""" """Checks if specified URL is cached."""
@ -62,14 +84,6 @@ class Cache:
except TypeError: except TypeError:
return False return False
def last_updated(self):
"""Returns *NIX timestamp of last update of the cache."""
return self.cache["last_update"]
def mark_updated(self):
"""Mark Cache as updated at current *NIX timestamp"""
self.cache["last_update"] = timestamp()
def last_modified(self, url): def last_modified(self, url):
"""Returns saved 'Last-Modified' header, if available.""" """Returns saved 'Last-Modified' header, if available."""
try: try:

77
twtxt/cli.py
View File

@ -12,7 +12,7 @@ import logging
import os import os
import sys import sys
import textwrap import textwrap
from time import time as timestamp from itertools import chain
import click import click
@ -99,21 +99,26 @@ def tweet(ctx, created_at, twtfile, text):
@click.option("--twtfile", "-f", @click.option("--twtfile", "-f",
type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True), type=click.Path(exists=True, file_okay=True, readable=True, resolve_path=True),
help="Location of your twtxt file. (Default: twtxt.txt") help="Location of your twtxt file. (Default: twtxt.txt")
@click.option("--ascending", "sorting", flag_value="ascending", @click.option("--ascending", "sorting",
flag_value="ascending",
help="Sort timeline in ascending order.") help="Sort timeline in ascending order.")
@click.option("--descending", "sorting", flag_value="descending", @click.option("--descending", "sorting",
flag_value="descending",
help="Sort timeline in descending order. (Default)") help="Sort timeline in descending order. (Default)")
@click.option("--timeout", type=click.FLOAT, @click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)") help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True, @click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)") help="Style output in an easy-to-parse format. (Default: False)")
@click.option("--source", "-s", @click.option("--source", "-s",
help="Only show feed of the given source. (Can be nick or URL)") help="Only show feed of the given source. (Can be nick or URL)")
@click.option("--cache/--no-cache", @click.option("--cache/--no-cache",
is_flag=True, is_flag=True,
help="Cache remote twtxt files locally. (Default: True)") help="Cache remote twtxt files locally. (Default: True)")
@click.option("--force-update", is_flag=True, @click.option("--force-update",
help="Force update even when timeline is called multiple times in a short period of time") is_flag=True,
help="Force update even if cache is up-to-date. (Default: False)")
@click.pass_context @click.pass_context
def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update): def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, cache, force_update):
"""Retrieve your personal timeline.""" """Retrieve your personal timeline."""
@ -126,17 +131,22 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
else: else:
sources = ctx.obj["conf"].following sources = ctx.obj["conf"].following
tweets = [] if cache:
with Cache.discover() as cache: try:
force_update = force_update or timestamp() - cache.last_updated() >= ctx.obj["conf"].timeline_update_interval with Cache.discover(update_interval=ctx.obj["conf"].timeline_update_interval) as cache:
force_update = force_update or not cache.is_valid
if force_update: if force_update:
tweets = get_remote_tweets(sources, limit, timeout) tweets = get_remote_tweets(sources, limit, timeout, cache)
else:
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(
cache.update_interval))
# Behold, almighty list comprehensions! (I might have gone overboard here…)
tweets = list(chain.from_iterable([cache.get_tweets(source.url) for source in sources]))
except OSError as e:
logger.debug(e)
tweets = get_remote_tweets(sources, limit, timeout)
else: else:
logger.debug("Multiple calls to 'timeline' within {0} seconds. Skipping update".format(ctx.obj["conf"].timeline_update_interval)) tweets = get_remote_tweets(sources, limit, timeout)
with Cache.discover() as cache:
for source in sources:
tweets += cache.get_tweets(source.url)
if twtfile and not source: if twtfile and not source:
source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile) source = Source(ctx.obj["conf"].nick, ctx.obj["conf"].twturl, file=twtfile)
@ -160,17 +170,24 @@ def timeline(ctx, pager, limit, twtfile, sorting, timeout, porcelain, source, ca
@click.option("--limit", "-l", @click.option("--limit", "-l",
type=click.INT, type=click.INT,
help="Limit total number of shown tweets. (Default: 20)") help="Limit total number of shown tweets. (Default: 20)")
@click.option("--ascending", "sorting", flag_value="ascending", @click.option("--ascending", "sorting",
flag_value="ascending",
help="Sort timeline in ascending order.") help="Sort timeline in ascending order.")
@click.option("--descending", "sorting", flag_value="descending", @click.option("--descending", "sorting",
flag_value="descending",
help="Sort timeline in descending order. (Default)") help="Sort timeline in descending order. (Default)")
@click.option("--timeout", type=click.FLOAT, @click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)") help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True, @click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)") help="Style output in an easy-to-parse format. (Default: False)")
@click.option("--cache/--no-cache", @click.option("--cache/--no-cache",
is_flag=True, is_flag=True,
help="Cache remote twtxt files locally. (Default: True)") help="Cache remote twtxt files locally. (Default: True)")
@click.option("--force-update",
is_flag=True,
help="Force update even if cache is up-to-date. (Default: False)")
@click.argument("source") @click.argument("source")
@click.pass_context @click.pass_context
def view(ctx, **kwargs): def view(ctx, **kwargs):
@ -182,9 +199,11 @@ def view(ctx, **kwargs):
@click.option("--check/--no-check", @click.option("--check/--no-check",
is_flag=True, is_flag=True,
help="Check if source URL is valid and readable. (Default: True)") help="Check if source URL is valid and readable. (Default: True)")
@click.option("--timeout", type=click.FLOAT, @click.option("--timeout",
type=click.FLOAT,
help="Maximum time requests are allowed to take. (Default: 5.0)") help="Maximum time requests are allowed to take. (Default: 5.0)")
@click.option("--porcelain", is_flag=True, @click.option("--porcelain",
is_flag=True,
help="Style output in an easy-to-parse format. (Default: False)") help="Style output in an easy-to-parse format. (Default: False)")
@click.pass_context @click.pass_context
def following(ctx, check, timeout, porcelain): def following(ctx, check, timeout, porcelain):
@ -204,7 +223,8 @@ def following(ctx, check, timeout, porcelain):
@cli.command() @cli.command()
@click.argument("nick") @click.argument("nick")
@click.argument("url") @click.argument("url")
@click.option("--force", "-f", flag_value=True, @click.option("--force", "-f",
flag_value=True,
help="Force adding and overwriting nick") help="Force adding and overwriting nick")
@click.pass_context @click.pass_context
def follow(ctx, nick, url, force): def follow(ctx, nick, url, force):
@ -283,7 +303,8 @@ def quickstart():
twtfile = os.path.expanduser(twtfile) twtfile = os.path.expanduser(twtfile)
overwrite_check(twtfile) overwrite_check(twtfile)
disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared", default=False) disclose_identity = click.confirm("➤ Do you want to disclose your identity? Your nick and URL will be shared",
default=False)
click.echo() click.echo()
add_news = click.confirm("➤ Do you want to follow the twtxt news feed?", default=True) add_news = click.confirm("➤ Do you want to follow the twtxt news feed?", default=True)
@ -303,9 +324,11 @@ def quickstart():
@cli.command() @cli.command()
@click.argument("key", required=False, callback=validate_config_key) @click.argument("key", required=False, callback=validate_config_key)
@click.argument("value", required=False) @click.argument("value", required=False)
@click.option("--remove", flag_value=True, @click.option("--remove",
flag_value=True,
help="Remove given item") help="Remove given item")
@click.option("--edit", "-e", flag_value=True, @click.option("--edit", "-e",
flag_value=True,
help="Open config file in editor") help="Open config file in editor")
@click.pass_context @click.pass_context
def config(ctx, key, value, remove, edit): def config(ctx, key, value, remove, edit):

3
twtxt/config.py
View File

@ -163,7 +163,7 @@ class Config:
@property @property
def timeline_update_interval(self): def timeline_update_interval(self):
return self.cfg.getint("twtxt", "timeline_update_interval", fallback = 10) return self.cfg.getint("twtxt", "timeline_update_interval", fallback=10)
@property @property
def use_abs_time(self): def use_abs_time(self):
@ -245,6 +245,7 @@ class Config:
"timeout": self.timeout, "timeout": self.timeout,
"sorting": self.sorting, "sorting": self.sorting,
"porcelain": self.porcelain, "porcelain": self.porcelain,
"update_interval": self.timeline_update_interval,
} }
} }
return default_map return default_map

12
twtxt/twhttp.py
View File

@ -107,7 +107,7 @@ def process_sources_for_file(client, sources, limit, cache=None):
return sorted(g_tweets, reverse=True)[:limit] return sorted(g_tweets, reverse=True)[:limit]
def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True): def get_remote_tweets(sources, limit=None, timeout=5.0, cache=None):
conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True) conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True)
headers = generate_user_agent() headers = generate_user_agent()
with aiohttp.ClientSession(connector=conn, headers=headers) as client: with aiohttp.ClientSession(connector=conn, headers=headers) as client:
@ -116,15 +116,7 @@ def get_remote_tweets(sources, limit=None, timeout=5.0, use_cache=True):
def start_loop(client, sources, limit, cache=None): def start_loop(client, sources, limit, cache=None):
return loop.run_until_complete(process_sources_for_file(client, sources, limit, cache)) return loop.run_until_complete(process_sources_for_file(client, sources, limit, cache))
if use_cache: tweets = start_loop(client, sources, limit, cache)
try:
with Cache.discover() as cache:
tweets = start_loop(client, sources, limit, cache)
except OSError as e:
logger.debug(e)
tweets = start_loop(client, sources, limit)
else:
tweets = start_loop(client, sources, limit)
return tweets return tweets