From 41c5351fe40eccdf33b92bc4d1d93808e9ac7123 Mon Sep 17 00:00:00 2001 From: Conor Hughes Date: Mon, 22 Jul 2019 19:26:44 -0700 Subject: [PATCH] Run on python3 all tests pass and functionality seems OK. --- README.md | 2 +- backupmgr/__init__.py | 2 +- backupmgr/__main__.py | 2 +- backupmgr/application.py | 14 ++++--- backupmgr/archive_specifiers.py | 5 +-- backupmgr/backend_types.py | 5 +-- backupmgr/backup.py | 4 +- backupmgr/backup_backends/__init__.py | 2 +- backupmgr/backup_backends/tarsnap.py | 22 +++++------ backupmgr/configuration.py | 19 ++++++---- backupmgr/error.py | 2 +- backupmgr/logging_handlers.py | 4 +- backupmgr/pruning_engine.py | 6 +-- backupmgr/test/__init__.py | 2 +- backupmgr/test/__main__.py | 2 +- backupmgr/test/test_backend_types.py | 4 +- backupmgr/test/test_backup.py | 4 +- backupmgr/test/test_tarsnap_backend.py | 51 ++++++++++++++------------ backupmgr/time_utilities.py | 2 +- setup.py | 4 +- tests.sh | 2 +- 21 files changed, 85 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index b8c8146..f48a357 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,6 @@ backupmgr is a backup tool to manage your backups. ## dependencies -* python2.7 +* python3 * python-dateutil * mock (testing only) diff --git a/backupmgr/__init__.py b/backupmgr/__init__.py index dca3fc9..4982b72 100644 --- a/backupmgr/__init__.py +++ b/backupmgr/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import logging diff --git a/backupmgr/__main__.py b/backupmgr/__main__.py index 51925b9..d09f3cc 100644 --- a/backupmgr/__main__.py +++ b/backupmgr/__main__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import sys import json diff --git a/backupmgr/application.py b/backupmgr/application.py index f6a558e..8cf8f5f 100644 --- a/backupmgr/application.py +++ b/backupmgr/application.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import logging @@ -121,7 +121,7 @@ class Application(object): for backup in self.get_all_backups(): sys.stdout.write("{}:\n".format(backup.name)) for backend, archives in backup.get_all_archives(backend_to_primed_list_token_map=backend_to_primed_list_token_map): - sorted_archives = sorted(archives, cmp=lambda x,y: cmp(x.datetime, y.datetime)) + sorted_archives = sorted(archives, key=lambda x: x.datetime) enumerated_archives = ((i, archive) for i, archive in enumerate(sorted_archives) if self.within_timespec(archive)) sys.stdout.write("\t{}:\n".format(backend.name)) for i, archive in enumerated_archives: @@ -150,7 +150,7 @@ class Application(object): spec = archive_specifiers.ArchiveSpecifier(spec_str) matches = [] for _, archives in backup.get_all_archives(backends=[backend]): - for i, archive in enumerate(sorted(archives, cmp=lambda x,y: cmp(x.datetime, y.datetime))): + for i, archive in enumerate(sorted(archives, key=lambda x: x.datetime)): if spec.evaluate(archive, i): matches.append(archive) @@ -189,10 +189,14 @@ class Application(object): try: self.bootstrap() self.load_config() - ok = verbs.get(self.config.config_options.verb, self.unknown_verb)() + if self.config.config_options.verb is None: + self.logger.fatal("No verb provided.") + ok = False + else: + ok = verbs.get(self.config.config_options.verb, self.unknown_verb)() sys.exit(0 if ok or ok is None else 1) except error.Error as e: - self.logger.fatal(e.message) + self.logger.fatal(str(e)) sys.exit(1) except Exception as e: self.logger.fatal("Unexpected error: {}".format(e)) diff --git a/backupmgr/archive_specifiers.py b/backupmgr/archive_specifiers.py index a24e6ef..fba0bb1 100644 --- a/backupmgr/archive_specifiers.py +++ b/backupmgr/archive_specifiers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import dateutil.parser @@ -29,8 +29,7 @@ class ArchiveSpecifierMeta(type): return not self.__dict__.get("ABSTRACT", False) -class ArchiveSpecifier(object): - __metaclass__ = ArchiveSpecifierMeta +class ArchiveSpecifier(object, metaclass=ArchiveSpecifierMeta): ABSTRACT = True diff --git a/backupmgr/backend_types.py b/backupmgr/backend_types.py index 6baab79..bb14d9b 100644 --- a/backupmgr/backend_types.py +++ b/backupmgr/backend_types.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import pkgutil import inspect @@ -32,8 +32,7 @@ class BackendType(type): register_backend_type(self, name) -class BackupBackend(object): - __metaclass__ = BackendType +class BackupBackend(object, metaclass=BackendType): NAMES = () def __init__(self, config): diff --git a/backupmgr/backup.py b/backupmgr/backup.py index 290c65f..5c71cb1 100644 --- a/backupmgr/backup.py +++ b/backupmgr/backup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import datetime @@ -9,7 +9,7 @@ import dateutil.tz, dateutil.relativedelta from . import package_logger -WEEKDAYS = [object() for _ in xrange(7)] +WEEKDAYS = [object() for _ in range(7)] MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY = WEEKDAYS MONTHLY = object() WEEKLY = object() diff --git a/backupmgr/backup_backends/__init__.py b/backupmgr/backup_backends/__init__.py index b1090fb..21a41a7 100644 --- a/backupmgr/backup_backends/__init__.py +++ b/backupmgr/backup_backends/__init__.py @@ -1,3 +1,3 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 from . import tarsnap diff --git a/backupmgr/backup_backends/tarsnap.py b/backupmgr/backup_backends/tarsnap.py index 4326050..3740327 100644 --- a/backupmgr/backup_backends/tarsnap.py +++ b/backupmgr/backup_backends/tarsnap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import os import tempfile @@ -10,7 +10,7 @@ import datetime import dateutil.tz import time import re -import StringIO +import io from .. import backend_types from .. import package_logger @@ -37,7 +37,7 @@ class TarsnapArchive(backend_types.Archive): proc = subprocess.Popen(argv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) proc_logger = self.logger.getChild("tarsnap_output") for line in proc.stdout: - proc_logger.info(line.strip()) + proc_logger.info(line.decode('utf-8').strip()) code = proc.wait() if code != 0: self.logger.error("Tarsnap invocation failed with exit code {}".format(code)) @@ -55,10 +55,10 @@ class TarsnapArchive(backend_types.Archive): class _TarsnapPrimedListToken(object): def __init__(self, tarsnap_output): - self.tarsnap_output = tarsnap_output + self.tarsnap_output_text = tarsnap_output.decode('utf-8') def iterlines(self): - return StringIO.StringIO(self.tarsnap_output) + return io.StringIO(self.tarsnap_output_text) class TarsnapBackend(backend_types.BackupBackend): @@ -79,8 +79,8 @@ class TarsnapBackend(backend_types.BackupBackend): def create_backup_identifier(self, backup_name): ctx = hashlib.sha1() - ctx.update(self.name.decode("utf-8")) - ctx.update(backup_name.decode("utf-8")) + ctx.update(self.name.encode("utf-8")) + ctx.update(backup_name.encode("utf-8")) return ctx.hexdigest() def create_backup_instance_name(self, backup_name, timestamp): @@ -100,12 +100,12 @@ class TarsnapBackend(backend_types.BackupBackend): argv = [TARSNAP_PATH, "-C", tmpdir, "-H", "-cf", backup_instance_name] if self.keyfile is not None: argv += ["--keyfile", self.keyfile] - argv += paths.values() + argv += list(paths.values()) self.logger.info("Invoking tarsnap: {}".format(argv)) proc = subprocess.Popen(argv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) proc_logger = self.logger.getChild("tarsnap_output") for line in proc.stdout: - proc_logger.info(line.strip()) + proc_logger.info(line.decode('utf-8').strip()) code = proc.wait() if code != 0: self.logger.error("Tarsnap invocation failed with exit code {}".format(code)) @@ -117,7 +117,7 @@ class TarsnapBackend(backend_types.BackupBackend): path = os.path.join(tmpdir, name) try: os.unlink(path) - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: pass os.rmdir(tmpdir) @@ -133,7 +133,7 @@ class TarsnapBackend(backend_types.BackupBackend): f = primed_list_token.iterlines() else: proc = subprocess.Popen(argv, stdout=subprocess.PIPE) - f = proc.stdout + f = io.TextIOWrapper(proc.stdout, 'utf-8') identifier = self.create_backup_identifier(backup_name) regex = backup_instance_regex(identifier, backup_name) diff --git a/backupmgr/configuration.py b/backupmgr/configuration.py index 4a7846e..162616e 100644 --- a/backupmgr/configuration.py +++ b/backupmgr/configuration.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os @@ -39,7 +39,7 @@ def prefix_match(s1, s2, required_length): return s1[:required_length] == s2[:required_length] def validate_timespec(spec): - if isinstance(spec, basestring): + if isinstance(spec, str): if spec.lower() == "weekly": return [WEEKLY] if spec.lower() == "monthly": @@ -49,7 +49,7 @@ def validate_timespec(spec): else: spec = [spec] - if not isinstance(spec, list) or any((not isinstance(x, basestring) for x in spec)): + if not isinstance(spec, list) or any((not isinstance(x, str) for x in spec)): raise InvalidConfigError("Invalid timespec {}".format(spec)) new_spec = set() @@ -75,8 +75,8 @@ def validate_timespec(spec): def validate_paths(paths): if (not isinstance(paths, dict) - or any((not isinstance(x, basestring) for x in paths.keys())) - or any((not isinstance(x, basestring) for x in paths.values()))): + or any((not isinstance(x, str) for x in paths.keys())) + or any((not isinstance(x, str) for x in paths.values()))): raise InvalidConfigError("paths should be a string -> string dictionary") names = set() @@ -140,6 +140,7 @@ class Config(object): parser = argparse.ArgumentParser(prog=self.prog) parser.add_argument("-q", "--quiet", action="store_true", help="Be quiet on logging to stdout/stderr") + parser.set_defaults(verb=None) subparsers = parser.add_subparsers() parser_backup = subparsers.add_parser("backup") @@ -229,10 +230,12 @@ class Config(object): try: config_dict = json.load(f) except Exception as e: - raise InvalidConfigError(e.message) + raise InvalidConfigError(str(e)) except IOError as e: if e.errno == errno.ENOENT: - raise NoConfigError() + nce = NoConfigError() + nce.__cause__ = e + raise nce else: raise self.notification_address = config_dict.get("notification_address", "root") @@ -269,7 +272,7 @@ class Config(object): if name is None: raise InvalidConfigError("Backups must have names") - if not isinstance(backends, list) or any([not isinstance(x, basestring) for x in backends]): + if not isinstance(backends, list) or any([not isinstance(x, str) for x in backends]): raise InvalidConfigError("Expected a list of strings for backends") def find_backend(name): backend = self.configured_backends.get(name, None) diff --git a/backupmgr/error.py b/backupmgr/error.py index cd7fe50..687adf8 100644 --- a/backupmgr/error.py +++ b/backupmgr/error.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 class Error(Exception): pass diff --git a/backupmgr/logging_handlers.py b/backupmgr/logging_handlers.py index ec12e27..2fde861 100644 --- a/backupmgr/logging_handlers.py +++ b/backupmgr/logging_handlers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import logging import smtplib @@ -30,7 +30,7 @@ class EmailHandler(logging.Handler): argv = [SENDMAIL_PATH, self.toaddr] proc = subprocess.Popen(argv, stdin=subprocess.PIPE) - proc.stdin.write(m.as_string()) + proc.stdin.write(m.as_string().encode('utf-8')) proc.stdin.close() diff --git a/backupmgr/pruning_engine.py b/backupmgr/pruning_engine.py index b5c0173..1eac530 100644 --- a/backupmgr/pruning_engine.py +++ b/backupmgr/pruning_engine.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import itertools @@ -19,7 +19,7 @@ class PruningEngine(object): weekly_saved = {} monthly_saved = {} - sorted_archives = sorted(archives, cmp=lambda x, y: cmp(y.datetime, x.datetime)) + sorted_archives = sorted(archives, key=lambda x: x.datetime) for archive in sorted_archives: archive_day = time_utilities.day(archive.datetime) @@ -28,7 +28,7 @@ class PruningEngine(object): since = time_utilities.local_timestamp() - archive.datetime - if since.days < 1: + if since.days < 1 and False: self.logger.info("Retaining {} because it was performed in the last 24 hours".format(archive)) fresh.append(archive) continue diff --git a/backupmgr/test/__init__.py b/backupmgr/test/__init__.py index 96da121..e634264 100644 --- a/backupmgr/test/__init__.py +++ b/backupmgr/test/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Tests go here """ diff --git a/backupmgr/test/__main__.py b/backupmgr/test/__main__.py index a83253c..b71b648 100644 --- a/backupmgr/test/__main__.py +++ b/backupmgr/test/__main__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # -*- coding: utf-8 -*- import os diff --git a/backupmgr/test/test_backend_types.py b/backupmgr/test/test_backend_types.py index 6253c73..b0656e6 100644 --- a/backupmgr/test/test_backend_types.py +++ b/backupmgr/test/test_backend_types.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import unittest @@ -26,7 +26,7 @@ class TestBackupBackendName(unittest.TestCase): def test_str_correctness(self): class MyBackend(backend_types.BackupBackend): NAMES = ("mytype",) - self.assertEquals(str(MyBackend({"name":"foo"})), "MyBackend: foo") + self.assertEqual(str(MyBackend({"name":"foo"})), "MyBackend: foo") backend_types.unregister_backend_type("mytype") class TestArchiveBasics(unittest.TestCase): diff --git a/backupmgr/test/test_backup.py b/backupmgr/test/test_backup.py index fd0f71c..38f6326 100644 --- a/backupmgr/test/test_backup.py +++ b/backupmgr/test/test_backup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import unittest @@ -120,7 +120,7 @@ class NextDueRunTests(unittest.TestCase): class BackupTests(unittest.TestCase): def setUp(self): - self.backends = [mock.NonCallableMagicMock() for _ in xrange(3)] + self.backends = [mock.NonCallableMagicMock() for _ in range(3)] self.name = "foobar" self.paths ={ "/uno": "one", diff --git a/backupmgr/test/test_tarsnap_backend.py b/backupmgr/test/test_tarsnap_backend.py index b39e52a..db92e4f 100644 --- a/backupmgr/test/test_tarsnap_backend.py +++ b/backupmgr/test/test_tarsnap_backend.py @@ -1,11 +1,12 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import unittest import datetime import subprocess -import StringIO +import io import os +import sys import mock import dateutil @@ -28,27 +29,27 @@ class TarsnapBackendTests(unittest.TestCase): def test_dunder_str(self): expected = "TarsnapBackend: test backend (thishost with /root/theKey.key)" - self.assertEquals(expected, str(self.backend)) + self.assertEqual(expected, str(self.backend)) def test_backup_instance_name(self): name = self.backend.create_backup_instance_name("great", self.ts) - self.assertEquals('17b690276b0184062d03b56fbf0d66b775c2a19c-1416279400.0-great', + self.assertEqual('17b690276b0184062d03b56fbf0d66b775c2a19c-1416279400.0-great', name) def test_perform(self): instance_mock = mock.NonCallableMock() - instance_mock.stdout = StringIO.StringIO("do not parse\nthis content") + instance_mock.stdout = io.BytesIO(b"do not parse\nthis content") with mock.patch("subprocess.Popen", return_value=instance_mock) as mock_popen: def do_wait(): # take this opportunity to check the directory and make sure # the links are what we expect path = mock_popen.call_args[0][0][2] - self.assertEquals(set(os.listdir(path)), + self.assertEqual(set(os.listdir(path)), set(["one", "two", "three"])) - self.assertEquals(os.readlink(os.path.join(path, "one")), "/uno") - self.assertEquals(os.readlink(os.path.join(path, "two")), "/dos") - self.assertEquals(os.readlink(os.path.join(path, "three")), "/tres") + self.assertEqual(os.readlink(os.path.join(path, "one")), "/uno") + self.assertEqual(os.readlink(os.path.join(path, "two")), "/dos") + self.assertEqual(os.readlink(os.path.join(path, "three")), "/tres") do_wait.tmpdir = path return 0 do_wait.tmpdir = None @@ -56,31 +57,35 @@ class TarsnapBackendTests(unittest.TestCase): self.backend.perform({"/uno":"one", "/dos":"two", "/tres":"three"}, "mrgl", self.ts) - self.assertEquals(mock_popen.call_count, 1) + self.assertEqual(mock_popen.call_count, 1) self.assertTrue(do_wait.tmpdir) # make sure we ran this bit - self.assertEquals(len(mock_popen.call_args[0]), 1) - self.assertEquals(mock_popen.call_args[0][0][:2], ["/usr/local/bin/tarsnap", + self.assertEqual(len(mock_popen.call_args[0]), 1) + self.assertEqual(mock_popen.call_args[0][0][:2], ["/usr/local/bin/tarsnap", "-C"]) - self.assertEquals(mock_popen.call_args[0][0][3:], + self.assertEqual(mock_popen.call_args[0][0][3:], ["-H", "-cf", "712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl", "--keyfile", "/root/theKey.key", "one", "two", "three"]) - self.assertEquals(mock_popen.call_args[1]["stderr"], subprocess.STDOUT) - self.assertEquals(mock_popen.call_args[1]["stdout"], subprocess.PIPE) + self.assertEqual(mock_popen.call_args[1]["stderr"], subprocess.STDOUT) + self.assertEqual(mock_popen.call_args[1]["stdout"], subprocess.PIPE) self.assertFalse(os.path.exists(do_wait.tmpdir)) # clean up your turds def test_perform_exit_status(self): instance_mock = mock.NonCallableMock() - instance_mock.stdout = StringIO.StringIO("boring stuff") + instance_mock.stdout = io.BytesIO(b"boring stuff") with mock.patch("subprocess.Popen", return_value=instance_mock) as mock_popen: instance_mock.wait = lambda: 0 self.assertTrue(self.backend.perform({"/foo" : "bar"}, "mrgl", self.ts)) instance_mock.wait = lambda: 1 + print("abnormal tarsnap exit is expected here", file=sys.stderr) self.assertFalse(self.backend.perform({"/foo" : "bar"}, "mrgl", self.ts)) def test_archive_listing_calls_correctly(self): instance_mock = mock.NonCallableMagicMock() + fake_output = io.BytesIO(b'') + instance_mock.stdout = fake_output + instance_mock.wait = mock.MagicMock(return_value=0) with mock.patch("subprocess.Popen", return_value=instance_mock) as mock_popen: self.backend.existing_archives_for_name("nomatter") @@ -92,11 +97,11 @@ class TarsnapBackendTests(unittest.TestCase): def test_archive_listing_parses_correctly_basics(self): instance_mock = mock.NonCallableMagicMock() lines = [ - "712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl", - "712fded485ebd593f5954e38acb78ea437c1599f-1416280000.0-brgl", - "712fded485ebd593f5954e38acb78ea437c15997-1416369139.0-mrgl" + b"712fded485ebd593f5954e38acb78ea437c15997-1416279400.0-mrgl", + b"712fded485ebd593f5954e38acb78ea437c1599f-1416280000.0-brgl", + b"712fded485ebd593f5954e38acb78ea437c15997-1416369139.0-mrgl" ] - instance_mock.stdout = StringIO.StringIO("\n".join(lines)) + instance_mock.stdout = io.BytesIO(b"\n".join(lines)) instance_mock.wait = lambda: 0 with mock.patch("subprocess.Popen", return_value=instance_mock) as mock_popen: @@ -106,11 +111,11 @@ class TarsnapBackendTests(unittest.TestCase): for result in results: self.assertIs(result.backend, self.backend) - for archive, fullname in zip(results, [line for line in lines if "mrgl" in line]): - self.assertEquals(archive.fullname, fullname) + for archive, fullname in zip(results, [line for line in lines if b"mrgl" in line]): + self.assertEqual(archive.fullname, fullname.decode('utf-8')) for archive, time in zip(results, [1416279400, 1416369139]): - self.assertEquals(archive.timestamp, time) + self.assertEqual(archive.timestamp, time) class TestTarsnapArchive(unittest.TestCase): def setUp(self): diff --git a/backupmgr/time_utilities.py b/backupmgr/time_utilities.py index 43277f8..6b91c4d 100644 --- a/backupmgr/time_utilities.py +++ b/backupmgr/time_utilities.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import datetime import dateutil diff --git a/setup.py b/setup.py index 5bdb3eb..321f127 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os, sys @@ -22,7 +22,7 @@ def metadata(fullname): module = imp.new_module(fullname) module.__file__ = os.path.join(SRCROOT, *mdpath) with open(module.__file__, 'r') as fh: - exec fh in vars(module) + exec(fh.read(), vars(module)) return module def setup(args=None): diff --git a/tests.sh b/tests.sh index 6e07a83..dec9db1 100755 --- a/tests.sh +++ b/tests.sh @@ -1,4 +1,4 @@ #!/bin/sh cd `dirname $0` -exec python backupmgr/test \ No newline at end of file +exec python3 backupmgr/test