Add "prune" verb
This commit is contained in:
Conor Hughes
parent
85ed230db2
commit
f94c33872f
|
|
@ -15,6 +15,7 @@ from . import error
|
|||
from . import backend_types
|
||||
from . import logging_handlers
|
||||
from . import archive_specifiers
|
||||
from . import pruning_engine
|
||||
|
||||
def pretty_archive(archive):
|
||||
local_time = archive.datetime.astimezone(dateutil.tz.tzlocal())
|
||||
|
|
@ -104,14 +105,19 @@ class Application(object):
|
|||
self.note_successful_backups(backup_successes)
|
||||
self.logger.info("Successfully completed {}/{} backups.".format(len(backup_successes), len(backups)))
|
||||
|
||||
def list_archives(self):
|
||||
def get_backend_to_primed_list_token_map(self):
|
||||
backend_to_primed_list_token_map = {}
|
||||
for backup in self.get_all_backups():
|
||||
for backend in backup.backends:
|
||||
if backend not in backend_to_primed_list_token_map:
|
||||
token = backend.get_primed_list_token()
|
||||
backend_to_primed_list_token_map[backend] = token
|
||||
|
||||
return backend_to_primed_list_token_map
|
||||
|
||||
|
||||
def list_archives(self):
|
||||
backend_to_primed_list_token_map = self.get_backend_to_primed_list_token_map()
|
||||
|
||||
for backup in self.get_all_backups():
|
||||
sys.stdout.write("{}:\n".format(backup.name))
|
||||
for backend, archives in backup.get_all_archives(backend_to_primed_list_token_map=backend_to_primed_list_token_map):
|
||||
|
|
@ -159,6 +165,15 @@ class Application(object):
|
|||
archive = matches[0]
|
||||
return archive.restore(self.config.config_options.destination)
|
||||
|
||||
def prune_archives(self):
|
||||
backend_to_primed_list_token_map = self.get_backend_to_primed_list_token_map()
|
||||
for backup in self.get_all_backups():
|
||||
pruning_config = self.config.pruning_configuration.get_backup_pruning_config(backup.name)
|
||||
for backend, archives in backup.get_all_archives(backend_to_primed_list_token_map=backend_to_primed_list_token_map):
|
||||
engine = pruning_engine.PruningEngine(pruning_config)
|
||||
archives_to_prune = engine.prunable_archives(archives)
|
||||
engine.prune_archives(archives_to_prune)
|
||||
|
||||
def unknown_verb(self):
|
||||
raise Exception("Unknown verb")
|
||||
|
||||
|
|
@ -168,7 +183,8 @@ class Application(object):
|
|||
"list": self.list_archives,
|
||||
"list-configured-backups": self.list_configured_backups,
|
||||
"list-backends": self.list_backends,
|
||||
"restore": self.restore_backup
|
||||
"restore": self.restore_backup,
|
||||
"prune": self.prune_archives
|
||||
}
|
||||
try:
|
||||
self.bootstrap()
|
||||
|
|
|
|||
|
|
@ -51,5 +51,8 @@ class Archive(object):
|
|||
dt = dt.replace(tzinfo=dateutil.tz.tzlocal())
|
||||
return dt
|
||||
|
||||
def __str__(self):
|
||||
return "'{}' with {} at {}".format(self.backup_name, self.backend.name, self.datetime)
|
||||
|
||||
def load_backend_types():
|
||||
from . import backup_backends
|
||||
|
|
|
|||
|
|
@ -27,13 +27,13 @@ class TarsnapArchive(backend_types.Archive):
|
|||
def logger(self):
|
||||
return package_logger().getChild("tarsnap_archive")
|
||||
|
||||
def __init__(self, backend, timestamp, fullname):
|
||||
def __init__(self, backend, timestamp, fullname, backup_name):
|
||||
self.fullname = fullname
|
||||
self.backend = backend
|
||||
self.timestamp = timestamp
|
||||
self.backup_name = backup_name
|
||||
|
||||
def restore(self, destination):
|
||||
argv = [TARSNAP_PATH, "-C", destination, "-x", "-f", self.fullname]
|
||||
def _invoke_tarsnap(self, argv):
|
||||
proc = subprocess.Popen(argv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
|
||||
proc_logger = self.logger.getChild("tarsnap_output")
|
||||
for line in proc.stdout:
|
||||
|
|
@ -44,6 +44,14 @@ class TarsnapArchive(backend_types.Archive):
|
|||
return False
|
||||
return True
|
||||
|
||||
def restore(self, destination):
|
||||
argv = [TARSNAP_PATH, "-C", destination, "-x", "-f", self.fullname]
|
||||
return self._invoke_tarsnap(argv)
|
||||
|
||||
def destroy(self):
|
||||
self.logger.info("destroying {}".format(self))
|
||||
argv = [TARSNAP_PATH, "-d", "-f", self.fullname]
|
||||
return self._invoke_tarsnap(argv)
|
||||
|
||||
class _TarsnapPrimedListToken(object):
|
||||
def __init__(self, tarsnap_output):
|
||||
|
|
@ -135,7 +143,7 @@ class TarsnapBackend(backend_types.BackupBackend):
|
|||
m = regex.match(line)
|
||||
if m:
|
||||
ts = float(m.groupdict()["timestamp"])
|
||||
results.append(TarsnapArchive(self, ts, m.group()))
|
||||
results.append(TarsnapArchive(self, ts, m.group(), backup_name))
|
||||
|
||||
if proc is not None and proc.wait() != 0:
|
||||
self.logger.error("Tarsnap invocation failed with exit code {}".format(proc.returncode))
|
||||
|
|
|
|||
|
|
@ -107,6 +107,20 @@ def parse_simple_date(datestr):
|
|||
return date
|
||||
|
||||
|
||||
BackupPruningConfiguration = collections.namedtuple(
|
||||
"BackupPruningConfiguration", ["backup_name", "daily_count", "weekly_count", "monthly_count"])
|
||||
|
||||
class PruningConfiguration(object):
|
||||
def __init__(self, backup_pruning_configs):
|
||||
self.__map = {}
|
||||
for config in backup_pruning_configs:
|
||||
self.__map[config.backup_name] = config
|
||||
|
||||
def get_backup_pruning_config(self, name):
|
||||
inf = float("inf")
|
||||
return self.__map.get(name, BackupPruningConfiguration(name, inf, inf, inf))
|
||||
|
||||
|
||||
class NoConfigError(error.Error):
|
||||
def __init__(self):
|
||||
super(NoConfigError, self).__init__("No config exists.")
|
||||
|
|
@ -151,6 +165,9 @@ class Config(object):
|
|||
parser_list_backends = subparsers.add_parser("list-backends")
|
||||
parser_list_backends.set_defaults(verb="list-backends")
|
||||
|
||||
parser_prune = subparsers.add_parser("prune")
|
||||
parser_prune.set_defaults(verb="prune")
|
||||
|
||||
return parser.parse_args(self.argv)
|
||||
|
||||
def default_state(self):
|
||||
|
|
@ -184,6 +201,22 @@ class Config(object):
|
|||
def all_configured_backends(self):
|
||||
return self.configured_backends.values()
|
||||
|
||||
def _parse_pruning_behavior(self, pruning_info):
|
||||
if not isinstance(pruning_info, dict):
|
||||
raise InvalidConfigError("Pruning info must be a dictionary")
|
||||
parsed_configs = []
|
||||
for name, config in pruning_info.items():
|
||||
if name not in [x.name for x in self.all_configured_backups()]:
|
||||
msg = "Attempt to define backup configuration for unknown backup {}".format(name)
|
||||
raise InvalidConfigError(msg)
|
||||
inf = float("inf")
|
||||
daily = config.get("daily", inf)
|
||||
weekly = config.get("weekly", inf)
|
||||
monthly = config.get("montly", inf)
|
||||
backup_config = BackupPruningConfiguration(name, daily, weekly, monthly)
|
||||
parsed_configs.append(backup_config)
|
||||
self.pruning_configuration = PruningConfiguration(parsed_configs)
|
||||
|
||||
def __init__(self, argv, prog):
|
||||
self.argv = argv
|
||||
self.prog = prog
|
||||
|
|
@ -270,3 +303,5 @@ class Config(object):
|
|||
self.configured_backups = backup.BackupSet(
|
||||
state, configured_backups, os.stat(self.configfile).st_mtime,
|
||||
state_mtime, datetime.datetime.now().replace(tzinfo=LOCAL_TZ))
|
||||
|
||||
self._parse_pruning_behavior(config_dict.get("pruning", {}))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env python2.7
|
||||
|
||||
import itertools
|
||||
|
||||
from . import time_utilities
|
||||
from . import package_logger
|
||||
|
||||
class PruningEngine(object):
|
||||
@property
|
||||
def logger(self):
|
||||
return package_logger().getChild("PruningEngine")
|
||||
|
||||
def __init__(self, pruning_config):
|
||||
self.pruning_config = pruning_config
|
||||
|
||||
def prunable_archives(self, archives):
|
||||
fresh = []
|
||||
daily_saved = {}
|
||||
weekly_saved = {}
|
||||
monthly_saved = {}
|
||||
|
||||
sorted_archives = sorted(archives, cmp=lambda x, y: cmp(y.datetime, x.datetime))
|
||||
|
||||
for archive in sorted_archives:
|
||||
archive_day = time_utilities.day(archive.datetime)
|
||||
archive_week = time_utilities.week(archive.datetime)
|
||||
archive_month = time_utilities.month(archive.datetime)
|
||||
|
||||
since = time_utilities.local_timestamp() - archive.datetime
|
||||
|
||||
if since.days < 1:
|
||||
self.logger.info("Retaining {} because it was performed in the last 24 hours".format(archive))
|
||||
fresh.append(archive)
|
||||
continue
|
||||
elif len(daily_saved) < self.pruning_config.daily_count and archive_day not in daily_saved:
|
||||
self.logger.info("Retaining {} as a daily backup".format(archive))
|
||||
daily_saved[archive_day] = archive
|
||||
elif len(weekly_saved) < self.pruning_config.weekly_count and archive_week not in weekly_saved:
|
||||
self.logger.info("Retaining {} as a weekly backup".format(archive))
|
||||
weekly_saved[archive_week] = archive
|
||||
elif len (monthly_saved) < self.pruning_config.monthly_count and archive_month not in monthly_saved:
|
||||
self.logger.info("Retaining {} as a monthly backup".format(archive))
|
||||
monthly_saved[archive_month] = archive
|
||||
|
||||
prunable = []
|
||||
saved_archives = set()
|
||||
for saved_archive in itertools.chain(fresh, daily_saved.values(),
|
||||
weekly_saved.values(),
|
||||
monthly_saved.values()):
|
||||
saved_archives.add(saved_archive)
|
||||
|
||||
return [archive for archive in sorted_archives if archive not in saved_archives]
|
||||
|
||||
def prune_archives(self, archives):
|
||||
for archive in archives:
|
||||
archive.destroy()
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env python2.7
|
||||
|
||||
import datetime
|
||||
import dateutil
|
||||
import dateutil.tz
|
||||
import dateutil.relativedelta
|
||||
|
||||
LOCAL_TZ = dateutil.tz.tzlocal()
|
||||
|
||||
def local_timestamp():
|
||||
return datetime.datetime.now().replace(tzinfo=LOCAL_TZ)
|
||||
|
||||
def day(dt):
|
||||
return dt.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
def week(dt):
|
||||
return (
|
||||
dt - dateutil.relativedelta.relativedelta(weekday=dateutil.relativedelta.MO(-1))
|
||||
).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
def month(dt):
|
||||
return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
Loading…
Reference in New Issue