conorh
/
backupmgr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run on python3 

all tests pass and functionality seems OK.
This commit is contained in:
Conor Hughes 2019-07-22 19:26:44 -07:00
parent d5ebcbe855
commit 41c5351fe4
21 changed files with 85 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -5,6 +5,6 @@
backupmgr is a backup tool to manage your backups.
## dependencies
* python2.7
* python3
* python-dateutil
* mock (testing only)

2
backupmgr/__init__.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import logging

2
backupmgr/__main__.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import sys
import json

14
backupmgr/application.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging
@ -121,7 +121,7 @@ class Application(object):
for backup in self.get_all_backups():
sys.stdout.write("{}:\n".format(backup.name))
for backend, archives in backup.get_all_archives(backend_to_primed_list_token_map=backend_to_primed_list_token_map):
sorted_archives = sorted(archives, cmp=lambda x,y: cmp(x.datetime, y.datetime))
sorted_archives = sorted(archives, key=lambda x: x.datetime)
enumerated_archives = ((i, archive) for i, archive in enumerate(sorted_archives) if self.within_timespec(archive))
sys.stdout.write("\t{}:\n".format(backend.name))
for i, archive in enumerated_archives:
@ -150,7 +150,7 @@ class Application(object):
spec = archive_specifiers.ArchiveSpecifier(spec_str)
matches = []
for _, archives in backup.get_all_archives(backends=[backend]):
for i, archive in enumerate(sorted(archives, cmp=lambda x,y: cmp(x.datetime, y.datetime))):
for i, archive in enumerate(sorted(archives, key=lambda x: x.datetime)):
if spec.evaluate(archive, i):
matches.append(archive)
@ -189,10 +189,14 @@ class Application(object):
try:
self.bootstrap()
self.load_config()
ok = verbs.get(self.config.config_options.verb, self.unknown_verb)()
if self.config.config_options.verb is None:
self.logger.fatal("No verb provided.")
ok = False
else:
ok = verbs.get(self.config.config_options.verb, self.unknown_verb)()
sys.exit(0 if ok or ok is None else 1)
except error.Error as e:
self.logger.fatal(e.message)
self.logger.fatal(str(e))
sys.exit(1)
except Exception as e:
self.logger.fatal("Unexpected error: {}".format(e))

5
backupmgr/archive_specifiers.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import dateutil.parser
@ -29,8 +29,7 @@ class ArchiveSpecifierMeta(type):
return not self.__dict__.get("ABSTRACT", False)
class ArchiveSpecifier(object):
__metaclass__ = ArchiveSpecifierMeta
class ArchiveSpecifier(object, metaclass=ArchiveSpecifierMeta):
ABSTRACT = True

5
backupmgr/backend_types.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import pkgutil
import inspect
@ -32,8 +32,7 @@ class BackendType(type):
register_backend_type(self, name)
class BackupBackend(object):
__metaclass__ = BackendType
class BackupBackend(object, metaclass=BackendType):
NAMES = ()
def __init__(self, config):

4
backupmgr/backup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import datetime
@ -9,7 +9,7 @@ import dateutil.tz, dateutil.relativedelta
from . import package_logger
WEEKDAYS = [object() for _ in xrange(7)]
WEEKDAYS = [object() for _ in range(7)]
MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY = WEEKDAYS
MONTHLY = object()
WEEKLY = object()

2
backupmgr/backup_backends/__init__.py
View File

@ -1,3 +1,3 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
from . import tarsnap

22
backupmgr/backup_backends/tarsnap.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import os
import tempfile
@ -10,7 +10,7 @@ import datetime
import dateutil.tz
import time
import re
import StringIO
import io
from .. import backend_types
from .. import package_logger
@ -37,7 +37,7 @@ class TarsnapArchive(backend_types.Archive):
proc = subprocess.Popen(argv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
proc_logger = self.logger.getChild("tarsnap_output")
for line in proc.stdout:
proc_logger.info(line.strip())
proc_logger.info(line.decode('utf-8').strip())
code = proc.wait()
if code != 0:
self.logger.error("Tarsnap invocation failed with exit code {}".format(code))
@ -55,10 +55,10 @@ class TarsnapArchive(backend_types.Archive):
class _TarsnapPrimedListToken(object):
def __init__(self, tarsnap_output):
self.tarsnap_output = tarsnap_output
self.tarsnap_output_text = tarsnap_output.decode('utf-8')
def iterlines(self):
return StringIO.StringIO(self.tarsnap_output)
return io.StringIO(self.tarsnap_output_text)
class TarsnapBackend(backend_types.BackupBackend):
@ -79,8 +79,8 @@ class TarsnapBackend(backend_types.BackupBackend):
def create_backup_identifier(self, backup_name):
ctx = hashlib.sha1()
ctx.update(self.name.decode("utf-8"))
ctx.update(backup_name.decode("utf-8"))
ctx.update(self.name.encode("utf-8"))
ctx.update(backup_name.encode("utf-8"))
return ctx.hexdigest()
def create_backup_instance_name(self, backup_name, timestamp):
@ -100,12 +100,12 @@ class TarsnapBackend(backend_types.BackupBackend):
argv = [TARSNAP_PATH, "-C", tmpdir, "-H", "-cf", backup_instance_name]
if self.keyfile is not None:
argv += ["--keyfile", self.keyfile]
argv += paths.values()
argv += list(paths.values())
self.logger.info("Invoking tarsnap: {}".format(argv))
proc = subprocess.Popen(argv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
proc_logger = self.logger.getChild("tarsnap_output")
for line in proc.stdout:
proc_logger.info(line.strip())
proc_logger.info(line.decode('utf-8').strip())
code = proc.wait()
if code != 0:
self.logger.error("Tarsnap invocation failed with exit code {}".format(code))
@ -117,7 +117,7 @@ class TarsnapBackend(backend_types.BackupBackend):
path = os.path.join(tmpdir, name)
try:
os.unlink(path)
except OSError, e:
except OSError as e:
if e.errno == errno.ENOENT:
pass
os.rmdir(tmpdir)
@ -133,7 +133,7 @@ class TarsnapBackend(backend_types.BackupBackend):
f = primed_list_token.iterlines()
else:
proc = subprocess.Popen(argv, stdout=subprocess.PIPE)
f = proc.stdout
f = io.TextIOWrapper(proc.stdout, 'utf-8')
identifier = self.create_backup_identifier(backup_name)
regex = backup_instance_regex(identifier, backup_name)

19
backupmgr/configuration.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
@ -39,7 +39,7 @@ def prefix_match(s1, s2, required_length):
return s1[:required_length] == s2[:required_length]
def validate_timespec(spec):
if isinstance(spec, basestring):
if isinstance(spec, str):
if spec.lower() == "weekly":
return [WEEKLY]
if spec.lower() == "monthly":
@ -49,7 +49,7 @@ def validate_timespec(spec):
else:
spec = [spec]
if not isinstance(spec, list) or any((not isinstance(x, basestring) for x in spec)):
if not isinstance(spec, list) or any((not isinstance(x, str) for x in spec)):
raise InvalidConfigError("Invalid timespec {}".format(spec))
new_spec = set()
@ -75,8 +75,8 @@ def validate_timespec(spec):
def validate_paths(paths):
if (not isinstance(paths, dict)
or any((not isinstance(x, basestring) for x in paths.keys()))
or any((not isinstance(x, basestring) for x in paths.values()))):
or any((not isinstance(x, str) for x in paths.keys()))
or any((not isinstance(x, str) for x in paths.values()))):
raise InvalidConfigError("paths should be a string -> string dictionary")
names = set()
@ -140,6 +140,7 @@ class Config(object):
parser = argparse.ArgumentParser(prog=self.prog)
parser.add_argument("-q", "--quiet", action="store_true",
help="Be quiet on logging to stdout/stderr")
parser.set_defaults(verb=None)
subparsers = parser.add_subparsers()
parser_backup = subparsers.add_parser("backup")
@ -229,10 +230,12 @@ class Config(object):
try:
config_dict = json.load(f)
except Exception as e:
raise InvalidConfigError(e.message)
raise InvalidConfigError(str(e))
except IOError as e:
if e.errno == errno.ENOENT:
raise NoConfigError()
nce = NoConfigError()
nce.__cause__ = e
raise nce
else:
raise
self.notification_address = config_dict.get("notification_address", "root")
@ -269,7 +272,7 @@ class Config(object):
if name is None:
raise InvalidConfigError("Backups must have names")
if not isinstance(backends, list) or any([not isinstance(x, basestring) for x in backends]):
if not isinstance(backends, list) or any([not isinstance(x, str) for x in backends]):
raise InvalidConfigError("Expected a list of strings for backends")
def find_backend(name):
backend = self.configured_backends.get(name, None)

2
backupmgr/error.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
class Error(Exception):
pass

4
backupmgr/logging_handlers.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import logging
import smtplib
@ -30,7 +30,7 @@ class EmailHandler(logging.Handler):
argv = [SENDMAIL_PATH, self.toaddr]
proc = subprocess.Popen(argv, stdin=subprocess.PIPE)
proc.stdin.write(m.as_string())
proc.stdin.write(m.as_string().encode('utf-8'))
proc.stdin.close()

6
backupmgr/pruning_engine.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import itertools
@ -19,7 +19,7 @@ class PruningEngine(object):
weekly_saved = {}
monthly_saved = {}
sorted_archives = sorted(archives, cmp=lambda x, y: cmp(y.datetime, x.datetime))
sorted_archives = sorted(archives, key=lambda x: x.datetime)
for archive in sorted_archives:
archive_day = time_utilities.day(archive.datetime)
@ -28,7 +28,7 @@ class PruningEngine(object):
since = time_utilities.local_timestamp() - archive.datetime
if since.days < 1:
if since.days < 1 and False:
self.logger.info("Retaining {} because it was performed in the last 24 hours".format(archive))
fresh.append(archive)
continue

2
backupmgr/test/__init__.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Tests go here """

2
backupmgr/test/__main__.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os

4
backupmgr/test/test_backend_types.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import unittest
@ -26,7 +26,7 @@ class TestBackupBackendName(unittest.TestCase):
def test_str_correctness(self):
class MyBackend(backend_types.BackupBackend):
NAMES = ("mytype",)
self.assertEquals(str(MyBackend({"name":"foo"})), "MyBackend: foo")
self.assertEqual(str(MyBackend({"name":"foo"})), "MyBackend: foo")
backend_types.unregister_backend_type("mytype")
class TestArchiveBasics(unittest.TestCase):

4
backupmgr/test/test_backup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import unittest
@ -120,7 +120,7 @@ class NextDueRunTests(unittest.TestCase):
class BackupTests(unittest.TestCase):
def setUp(self):
self.backends = [mock.NonCallableMagicMock() for _ in xrange(3)]
self.backends = [mock.NonCallableMagicMock() for _ in range(3)]
self.name = "foobar"
self.paths ={
"/uno": "one",

51
backupmgr/test/test_tarsnap_backend.py
View File

@ -1,11 +1,12 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import unittest
import datetime
import subprocess
import StringIO
import io
import os
import sys
import mock
import dateutil
@ -28,27 +29,27 @@ class TarsnapBackendTests(unittest.TestCase):
def test_dunder_str(self):
expected = "TarsnapBackend: test backend (thishost with /root/theKey.key)"
self.assertEquals(expected, str(self.backend))
self.assertEqual(expected, str(self.backend))
def test_backup_instance_name(self):
name = self.backend.create_backup_instance_name("great", self.ts)
self.assertEquals('17b690276b0184062d03b56fbf0d66b775c2a19c-1416279400.0-great',
self.assertEqual('17b690276b0184062d03b56fbf0d66b775c2a19c-1416279400.0-great',
name)
def test_perform(self):
instance_mock = mock.NonCallableMock()
instance_mock.stdout = StringIO.StringIO("do not parse\nthis content")
instance_mock.stdout = io.BytesIO(b"do not parse\nthis content")
with mock.patch("subprocess.Popen",
return_value=instance_mock) as mock_popen:
def do_wait():
# take this opportunity to check the directory and make sure
# the links are what we expect
path = mock_popen.call_args[0][0][2]
self.assertEquals(set(os.listdir(path)),
self.assertEqual(set(os.listdir(path)),
set(["one", "two", "three"]))
self.assertEquals(os.readlink(os.path.join(path, "one")), "/uno")
self.assertEquals(os.readlink(os.path.join(path, "two")), "/dos")
self.assertEquals(os.readlink(os.path.join(path, "three")), "/tres")
self.assertEqual(os.readlink(os.path.join(path, "one")), "/uno")
self.assertEqual(os.readlink(os.path.join(path, "two")), "/dos")
self.assertEqual(os.readlink(os.path.join(path, "three")), "/tres")
do_wait.tmpdir = path
return 0
do_wait.tmpdir = None
@ -56,31 +57,35 @@ class TarsnapBackendTests(unittest.TestCase):
self.backend.perform({"/uno":"one", "/dos":"two", "/tres":"three"},
"mrgl", self.ts)
self.assertEquals(mock_popen.call_count, 1)
self.assertEqual(mock_popen.call_count, 1)
self.assertTrue(do_wait.tmpdir) # make sure we ran this bit
self.assertEquals(len(mock_popen.call_args[0]), 1)
self.assertEquals(mock_popen.call_args[0][0][:2], ["/usr/local/bin/tarsnap",
self.assertEqual(len(mock_popen.call_args[0]), 1)
self.assertEqual(mock_popen.call_args[0][0][:2], ["/usr/local/bin/tarsnap",
"-C"])
self.assertEquals(mock_popen.call_args[0][0][3:],
self.assertEqual(mock_popen.call_args[0][0][3:],
["-H", "-cf",
"712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl",
"--keyfile", "/root/theKey.key", "one", "two", "three"])
self.assertEquals(mock_popen.call_args[1]["stderr"], subprocess.STDOUT)
self.assertEquals(mock_popen.call_args[1]["stdout"], subprocess.PIPE)
self.assertEqual(mock_popen.call_args[1]["stderr"], subprocess.STDOUT)
self.assertEqual(mock_popen.call_args[1]["stdout"], subprocess.PIPE)
self.assertFalse(os.path.exists(do_wait.tmpdir)) # clean up your turds
def test_perform_exit_status(self):
instance_mock = mock.NonCallableMock()
instance_mock.stdout = StringIO.StringIO("boring stuff")
instance_mock.stdout = io.BytesIO(b"boring stuff")
with mock.patch("subprocess.Popen",
return_value=instance_mock) as mock_popen:
instance_mock.wait = lambda: 0
self.assertTrue(self.backend.perform({"/foo" : "bar"}, "mrgl", self.ts))
instance_mock.wait = lambda: 1
print("abnormal tarsnap exit is expected here", file=sys.stderr)
self.assertFalse(self.backend.perform({"/foo" : "bar"}, "mrgl", self.ts))
def test_archive_listing_calls_correctly(self):
instance_mock = mock.NonCallableMagicMock()
fake_output = io.BytesIO(b'')
instance_mock.stdout = fake_output
instance_mock.wait = mock.MagicMock(return_value=0)
with mock.patch("subprocess.Popen",
return_value=instance_mock) as mock_popen:
self.backend.existing_archives_for_name("nomatter")
@ -92,11 +97,11 @@ class TarsnapBackendTests(unittest.TestCase):
def test_archive_listing_parses_correctly_basics(self):
instance_mock = mock.NonCallableMagicMock()
lines = [
"712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl",
"712fded485ebd593f5954e38acb78ea437c1599f-1416280000.0-brgl",
"712fded485ebd593f5954e38acb78ea437c15997-1416369139.0-mrgl"
b"712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl",
b"712fded485ebd593f5954e38acb78ea437c1599f-1416280000.0-brgl",
b"712fded485ebd593f5954e38acb78ea437c15997-1416369139.0-mrgl"
]
instance_mock.stdout = StringIO.StringIO("\n".join(lines))
instance_mock.stdout = io.BytesIO(b"\n".join(lines))
instance_mock.wait = lambda: 0
with mock.patch("subprocess.Popen",
return_value=instance_mock) as mock_popen:
@ -106,11 +111,11 @@ class TarsnapBackendTests(unittest.TestCase):
for result in results:
self.assertIs(result.backend, self.backend)
for archive, fullname in zip(results, [line for line in lines if "mrgl" in line]):
self.assertEquals(archive.fullname, fullname)
for archive, fullname in zip(results, [line for line in lines if b"mrgl" in line]):
self.assertEqual(archive.fullname, fullname.decode('utf-8'))
for archive, time in zip(results, [1416279400, 1416369139]):
self.assertEquals(archive.timestamp, time)
self.assertEqual(archive.timestamp, time)
class TestTarsnapArchive(unittest.TestCase):
def setUp(self):

2
backupmgr/time_utilities.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
import datetime
import dateutil

4
setup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, sys
@ -22,7 +22,7 @@ def metadata(fullname):
module = imp.new_module(fullname)
module.__file__ = os.path.join(SRCROOT, *mdpath)
with open(module.__file__, 'r') as fh:
exec fh in vars(module)
exec(fh.read(), vars(module))
return module
def setup(args=None):

2
tests.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh
cd `dirname $0`
exec python backupmgr/test
exec python3 backupmgr/test