From baf3afb25b8aa46d2fb6d091f6f656ac681bea4a Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Wed, 29 Mar 2023 11:15:38 +0000 Subject: [PATCH] refactor(backups): make backups stateless --- selfprivacy_api/backup/__init__.py | 88 ++++++++++++------- .../backup/providers/local_file.py | 4 +- selfprivacy_api/backup/providers/provider.py | 2 +- tests/test_graphql/test_backup.py | 64 +++++++------- 4 files changed, 92 insertions(+), 66 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 550555d..9ac8895 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -16,6 +16,9 @@ from selfprivacy_api.backup.providers.provider import AbstractBackupProvider from selfprivacy_api.backup.providers import get_provider, get_kind from selfprivacy_api.graphql.queries.providers import BackupProvider +# a hack to store file path. +REDIS_REPO_PATH_KEY = "backups:test_repo_path" + REDIS_PROVIDER_KEY = "backups:provider" REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:" @@ -30,20 +33,30 @@ class Backups: provider: AbstractBackupProvider - def __init__(self, test_repo_file: str = ""): - if test_repo_file != "": - self.set_localfile_repo(test_repo_file) - else: - self.provider = self.lookup_provider() - - def set_localfile_repo(self, file_path: str): + @staticmethod + def set_localfile_repo(file_path: str): ProviderClass = get_provider(BackupProvider.FILE) provider = ProviderClass(file_path) - self.provider = provider + redis.set(REDIS_REPO_PATH_KEY, file_path) + Backups.store_provider_redis(provider) + + @staticmethod + def provider(): + return Backups.lookup_provider() + + @staticmethod + def set_provider(kind: str, login: str, key: str): + provider = Backups.construct_provider(kind, login, key) + Backups.store_provider_redis(provider) @staticmethod def construct_provider(kind: str, login: str, key: str): provider_class = get_provider(BackupProvider[kind]) + + if kind == "FILE": + path = redis.get(REDIS_REPO_PATH_KEY) + return provider_class(path) + return provider_class(login=login, key=key) @staticmethod @@ -68,19 +81,24 @@ class Backups: @staticmethod def reset(): redis.delete(REDIS_PROVIDER_KEY) + redis.delete(REDIS_REPO_PATH_KEY) for key in redis.keys(REDIS_INITTED_CACHE_PREFIX + "*"): redis.delete(key) - def lookup_provider(self) -> AbstractBackupProvider: + @staticmethod + def lookup_provider() -> AbstractBackupProvider: redis_provider = Backups.load_provider_redis() if redis_provider is not None: return redis_provider json_provider = Backups.load_provider_json() if json_provider is not None: + Backups.store_provider_redis(json_provider) return json_provider - return Backups.construct_provider("MEMORY", login="", key="") + memory_provider = Backups.construct_provider("MEMORY", login="", key="") + Backups.store_provider_redis(memory_provider) + return memory_provider @staticmethod def load_provider_json() -> AbstractBackupProvider: @@ -105,64 +123,74 @@ class Backups: kind=provider_string, login=account, key=key ) - def back_up(self, service: Service): + @staticmethod + def back_up(service: Service): folder = service.get_location() repo_name = service.get_id() service.pre_backup() - self.provider.backuper.start_backup(folder, repo_name) + Backups.provider().backuper.start_backup(folder, repo_name) service.post_restore() - def init_repo(self, service: Service): + @staticmethod + def init_repo(service: Service): repo_name = service.get_id() - self.provider.backuper.init(repo_name) - self._redis_mark_as_init(service) + Backups.provider().backuper.init(repo_name) + Backups._redis_mark_as_init(service) - def _has_redis_init_mark(self, service: Service) -> bool: + @staticmethod + def _has_redis_init_mark(service: Service) -> bool: repo_name = service.get_id() if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name): return True return False - def _redis_mark_as_init(self, service: Service): + @staticmethod + def _redis_mark_as_init(service: Service): repo_name = service.get_id() redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1) - def is_initted(self, service: Service) -> bool: + @staticmethod + def is_initted(service: Service) -> bool: repo_name = service.get_id() - if self._has_redis_init_mark(service): + if Backups._has_redis_init_mark(service): return True - initted = self.provider.backuper.is_initted(repo_name) + initted = Backups.provider().backuper.is_initted(repo_name) if initted: - self._redis_mark_as_init(service) + Backups._redis_mark_as_init(service) return True return False - def get_snapshots(self, service: Service) -> List[Snapshot]: + @staticmethod + def get_snapshots(service: Service) -> List[Snapshot]: repo_name = service.get_id() - return self.provider.backuper.get_snapshots(repo_name) + return Backups.provider().backuper.get_snapshots(repo_name) - def restore_service_from_snapshot(self, service: Service, snapshot_id: str): + @staticmethod + def restore_service_from_snapshot(service: Service, snapshot_id: str): repo_name = service.get_id() folder = service.get_location() - self.provider.backuper.restore_from_backup(repo_name, snapshot_id, folder) + Backups.provider().backuper.restore_from_backup(repo_name, snapshot_id, folder) # Our dummy service is not yet globally registered so this is not testable yet - def restore_snapshot(self, snapshot: Snapshot): - self.restore_service_from_snapshot( + @staticmethod + def restore_snapshot(snapshot: Snapshot): + Backups.restore_service_from_snapshot( get_service_by_id(snapshot.service_name), snapshot.id ) - def service_snapshot_size(self, service: Service, snapshot_id: str) -> float: + @staticmethod + def service_snapshot_size(service: Service, snapshot_id: str) -> float: repo_name = service.get_id() - return self.provider.backuper.restored_size(repo_name, snapshot_id) + return Backups.provider().backuper.restored_size(repo_name, snapshot_id) # Our dummy service is not yet globally registered so this is not testable yet - def snapshot_restored_size(self, snapshot: Snapshot) -> float: + @staticmethod + def snapshot_restored_size(snapshot: Snapshot) -> float: return self.service_snapshot_size( get_service_by_id(snapshot.service_name), snapshot.id ) diff --git a/selfprivacy_api/backup/providers/local_file.py b/selfprivacy_api/backup/providers/local_file.py index 5ae45bd..bdd9213 100644 --- a/selfprivacy_api/backup/providers/local_file.py +++ b/selfprivacy_api/backup/providers/local_file.py @@ -5,5 +5,7 @@ from selfprivacy_api.backup.restic_backuper import ResticBackuper class LocalFileBackup(AbstractBackupProvider): backuper = ResticBackuper("", "", "memory") - def __init__(self, filename: str): + # login and key args are for compatibility with generic provider methods. They are ignored. + def __init__(self, filename: str, login: str = "", key: str = ""): + super().__init__() self.backuper = ResticBackuper("", "", f":local:{filename}/") diff --git a/selfprivacy_api/backup/providers/provider.py b/selfprivacy_api/backup/providers/provider.py index 0b57528..017c03d 100644 --- a/selfprivacy_api/backup/providers/provider.py +++ b/selfprivacy_api/backup/providers/provider.py @@ -12,6 +12,6 @@ class AbstractBackupProvider(ABC): def backuper(self) -> AbstractBackuper: raise NotImplementedError - def __init__(self, login, key): + def __init__(self, login="", key=""): self.login = login self.key = key diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 233014f..2e6c6d6 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -22,17 +22,15 @@ REPO_NAME = "test_backup" @pytest.fixture(scope="function") def backups(tmpdir): + Backups.reset() + test_repo_path = path.join(tmpdir, "totallyunrelated") - backups = Backups(test_repo_path) - backups.reset() - return backups + Backups.set_localfile_repo(test_repo_path) @pytest.fixture() def backups_backblaze(generic_userdata): - backups = Backups() - backups.reset() - return backups + Backups.reset() @pytest.fixture() @@ -59,7 +57,7 @@ def dummy_service(tmpdir, backups, raw_dummy_service): assert not path.exists(repo_path) # assert not repo_path - backups.init_repo(service) + Backups.init_repo(service) return service @@ -83,9 +81,8 @@ def file_backup(tmpdir) -> AbstractBackupProvider: def test_config_load(generic_userdata): - backups = Backups() - backups.reset() - provider = backups.provider + Backups.reset() + provider = Backups.provider() assert provider is not None assert isinstance(provider, Backblaze) @@ -114,7 +111,7 @@ def test_backup_simple_file(raw_dummy_service, file_backup): def test_backup_service(dummy_service, backups): - backups.back_up(dummy_service) + Backups.back_up(dummy_service) def test_no_repo(memory_backup): @@ -123,9 +120,9 @@ def test_no_repo(memory_backup): def test_one_snapshot(backups, dummy_service): - backups.back_up(dummy_service) + Backups.back_up(dummy_service) - snaps = backups.get_snapshots(dummy_service) + snaps = Backups.get_snapshots(dummy_service) assert len(snaps) == 1 snap = snaps[0] assert snap.service_name == dummy_service.get_id() @@ -137,30 +134,29 @@ def test_restore(backups, dummy_service): assert file_to_nuke is not None path_to_nuke = path.join(service_folder, file_to_nuke) - backups.back_up(dummy_service) - snap = backups.get_snapshots(dummy_service)[0] + Backups.back_up(dummy_service) + snap = Backups.get_snapshots(dummy_service)[0] assert snap is not None assert path.exists(path_to_nuke) remove(path_to_nuke) assert not path.exists(path_to_nuke) - backups.restore_service_from_snapshot(dummy_service, snap.id) + Backups.restore_service_from_snapshot(dummy_service, snap.id) assert path.exists(path_to_nuke) def test_sizing(backups, dummy_service): - backups.back_up(dummy_service) - snap = backups.get_snapshots(dummy_service)[0] - size = backups.service_snapshot_size(dummy_service, snap.id) + Backups.back_up(dummy_service) + snap = Backups.get_snapshots(dummy_service)[0] + size = Backups.service_snapshot_size(dummy_service, snap.id) assert size is not None assert size > 0 def test_redis_storage(backups_backblaze): - backups = Backups() - backups.reset() - provider = backups.provider + Backups.reset() + provider = Backups.provider() assert provider is not None @@ -168,8 +164,8 @@ def test_redis_storage(backups_backblaze): assert provider.login == "ID" assert provider.key == "KEY" - backups.store_provider_redis(provider) - restored_provider = backups.load_provider_redis() + Backups.store_provider_redis(provider) + restored_provider = Backups.load_provider_redis() assert isinstance(restored_provider, Backblaze) assert restored_provider.login == "ID" assert restored_provider.key == "KEY" @@ -177,27 +173,27 @@ def test_redis_storage(backups_backblaze): # lowlevel def test_init_tracking_caching(backups, raw_dummy_service): - assert backups._has_redis_init_mark(raw_dummy_service) is False + assert Backups._has_redis_init_mark(raw_dummy_service) is False - backups._redis_mark_as_init(raw_dummy_service) + Backups._redis_mark_as_init(raw_dummy_service) - assert backups._has_redis_init_mark(raw_dummy_service) is True - assert backups.is_initted(raw_dummy_service) is True + assert Backups._has_redis_init_mark(raw_dummy_service) is True + assert Backups.is_initted(raw_dummy_service) is True # lowlevel def test_init_tracking_caching2(backups, raw_dummy_service): - assert backups._has_redis_init_mark(raw_dummy_service) is False + assert Backups._has_redis_init_mark(raw_dummy_service) is False - backups.init_repo(raw_dummy_service) + Backups.init_repo(raw_dummy_service) - assert backups._has_redis_init_mark(raw_dummy_service) is True + assert Backups._has_redis_init_mark(raw_dummy_service) is True # only public API def test_init_tracking(backups, raw_dummy_service): - assert backups.is_initted(raw_dummy_service) is False + assert Backups.is_initted(raw_dummy_service) is False - backups.init_repo(raw_dummy_service) + Backups.init_repo(raw_dummy_service) - assert backups.is_initted(raw_dummy_service) is True + assert Backups.is_initted(raw_dummy_service) is True