refactor(backups): make backups stateless

restic-rewrite-api
Houkime 2023-03-29 11:15:38 +00:00
parent a2dd47130b
commit 144e4e5e91
4 changed files with 92 additions and 66 deletions

View File

@ -16,6 +16,9 @@ from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
from selfprivacy_api.backup.providers import get_provider, get_kind from selfprivacy_api.backup.providers import get_provider, get_kind
from selfprivacy_api.graphql.queries.providers import BackupProvider from selfprivacy_api.graphql.queries.providers import BackupProvider
# a hack to store file path.
REDIS_REPO_PATH_KEY = "backups:test_repo_path"
REDIS_PROVIDER_KEY = "backups:provider" REDIS_PROVIDER_KEY = "backups:provider"
REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:" REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:"
@ -30,20 +33,30 @@ class Backups:
provider: AbstractBackupProvider provider: AbstractBackupProvider
def __init__(self, test_repo_file: str = ""): @staticmethod
if test_repo_file != "": def set_localfile_repo(file_path: str):
self.set_localfile_repo(test_repo_file)
else:
self.provider = self.lookup_provider()
def set_localfile_repo(self, file_path: str):
ProviderClass = get_provider(BackupProvider.FILE) ProviderClass = get_provider(BackupProvider.FILE)
provider = ProviderClass(file_path) provider = ProviderClass(file_path)
self.provider = provider redis.set(REDIS_REPO_PATH_KEY, file_path)
Backups.store_provider_redis(provider)
@staticmethod
def provider():
return Backups.lookup_provider()
@staticmethod
def set_provider(kind: str, login: str, key: str):
provider = Backups.construct_provider(kind, login, key)
Backups.store_provider_redis(provider)
@staticmethod @staticmethod
def construct_provider(kind: str, login: str, key: str): def construct_provider(kind: str, login: str, key: str):
provider_class = get_provider(BackupProvider[kind]) provider_class = get_provider(BackupProvider[kind])
if kind == "FILE":
path = redis.get(REDIS_REPO_PATH_KEY)
return provider_class(path)
return provider_class(login=login, key=key) return provider_class(login=login, key=key)
@staticmethod @staticmethod
@ -68,19 +81,24 @@ class Backups:
@staticmethod @staticmethod
def reset(): def reset():
redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_PROVIDER_KEY)
redis.delete(REDIS_REPO_PATH_KEY)
for key in redis.keys(REDIS_INITTED_CACHE_PREFIX + "*"): for key in redis.keys(REDIS_INITTED_CACHE_PREFIX + "*"):
redis.delete(key) redis.delete(key)
def lookup_provider(self) -> AbstractBackupProvider: @staticmethod
def lookup_provider() -> AbstractBackupProvider:
redis_provider = Backups.load_provider_redis() redis_provider = Backups.load_provider_redis()
if redis_provider is not None: if redis_provider is not None:
return redis_provider return redis_provider
json_provider = Backups.load_provider_json() json_provider = Backups.load_provider_json()
if json_provider is not None: if json_provider is not None:
Backups.store_provider_redis(json_provider)
return json_provider return json_provider
return Backups.construct_provider("MEMORY", login="", key="") memory_provider = Backups.construct_provider("MEMORY", login="", key="")
Backups.store_provider_redis(memory_provider)
return memory_provider
@staticmethod @staticmethod
def load_provider_json() -> AbstractBackupProvider: def load_provider_json() -> AbstractBackupProvider:
@ -105,64 +123,74 @@ class Backups:
kind=provider_string, login=account, key=key kind=provider_string, login=account, key=key
) )
def back_up(self, service: Service): @staticmethod
def back_up(service: Service):
folder = service.get_location() folder = service.get_location()
repo_name = service.get_id() repo_name = service.get_id()
service.pre_backup() service.pre_backup()
self.provider.backuper.start_backup(folder, repo_name) Backups.provider().backuper.start_backup(folder, repo_name)
service.post_restore() service.post_restore()
def init_repo(self, service: Service): @staticmethod
def init_repo(service: Service):
repo_name = service.get_id() repo_name = service.get_id()
self.provider.backuper.init(repo_name) Backups.provider().backuper.init(repo_name)
self._redis_mark_as_init(service) Backups._redis_mark_as_init(service)
def _has_redis_init_mark(self, service: Service) -> bool: @staticmethod
def _has_redis_init_mark(service: Service) -> bool:
repo_name = service.get_id() repo_name = service.get_id()
if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name): if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name):
return True return True
return False return False
def _redis_mark_as_init(self, service: Service): @staticmethod
def _redis_mark_as_init(service: Service):
repo_name = service.get_id() repo_name = service.get_id()
redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1) redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1)
def is_initted(self, service: Service) -> bool: @staticmethod
def is_initted(service: Service) -> bool:
repo_name = service.get_id() repo_name = service.get_id()
if self._has_redis_init_mark(service): if Backups._has_redis_init_mark(service):
return True return True
initted = self.provider.backuper.is_initted(repo_name) initted = Backups.provider().backuper.is_initted(repo_name)
if initted: if initted:
self._redis_mark_as_init(service) Backups._redis_mark_as_init(service)
return True return True
return False return False
def get_snapshots(self, service: Service) -> List[Snapshot]: @staticmethod
def get_snapshots(service: Service) -> List[Snapshot]:
repo_name = service.get_id() repo_name = service.get_id()
return self.provider.backuper.get_snapshots(repo_name) return Backups.provider().backuper.get_snapshots(repo_name)
def restore_service_from_snapshot(self, service: Service, snapshot_id: str): @staticmethod
def restore_service_from_snapshot(service: Service, snapshot_id: str):
repo_name = service.get_id() repo_name = service.get_id()
folder = service.get_location() folder = service.get_location()
self.provider.backuper.restore_from_backup(repo_name, snapshot_id, folder) Backups.provider().backuper.restore_from_backup(repo_name, snapshot_id, folder)
# Our dummy service is not yet globally registered so this is not testable yet # Our dummy service is not yet globally registered so this is not testable yet
def restore_snapshot(self, snapshot: Snapshot): @staticmethod
self.restore_service_from_snapshot( def restore_snapshot(snapshot: Snapshot):
Backups.restore_service_from_snapshot(
get_service_by_id(snapshot.service_name), snapshot.id get_service_by_id(snapshot.service_name), snapshot.id
) )
def service_snapshot_size(self, service: Service, snapshot_id: str) -> float: @staticmethod
def service_snapshot_size(service: Service, snapshot_id: str) -> float:
repo_name = service.get_id() repo_name = service.get_id()
return self.provider.backuper.restored_size(repo_name, snapshot_id) return Backups.provider().backuper.restored_size(repo_name, snapshot_id)
# Our dummy service is not yet globally registered so this is not testable yet # Our dummy service is not yet globally registered so this is not testable yet
def snapshot_restored_size(self, snapshot: Snapshot) -> float: @staticmethod
def snapshot_restored_size(snapshot: Snapshot) -> float:
return self.service_snapshot_size( return self.service_snapshot_size(
get_service_by_id(snapshot.service_name), snapshot.id get_service_by_id(snapshot.service_name), snapshot.id
) )

View File

@ -5,5 +5,7 @@ from selfprivacy_api.backup.restic_backuper import ResticBackuper
class LocalFileBackup(AbstractBackupProvider): class LocalFileBackup(AbstractBackupProvider):
backuper = ResticBackuper("", "", "memory") backuper = ResticBackuper("", "", "memory")
def __init__(self, filename: str): # login and key args are for compatibility with generic provider methods. They are ignored.
def __init__(self, filename: str, login: str = "", key: str = ""):
super().__init__()
self.backuper = ResticBackuper("", "", f":local:{filename}/") self.backuper = ResticBackuper("", "", f":local:{filename}/")

View File

@ -12,6 +12,6 @@ class AbstractBackupProvider(ABC):
def backuper(self) -> AbstractBackuper: def backuper(self) -> AbstractBackuper:
raise NotImplementedError raise NotImplementedError
def __init__(self, login, key): def __init__(self, login="", key=""):
self.login = login self.login = login
self.key = key self.key = key

View File

@ -22,17 +22,15 @@ REPO_NAME = "test_backup"
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def backups(tmpdir): def backups(tmpdir):
Backups.reset()
test_repo_path = path.join(tmpdir, "totallyunrelated") test_repo_path = path.join(tmpdir, "totallyunrelated")
backups = Backups(test_repo_path) Backups.set_localfile_repo(test_repo_path)
backups.reset()
return backups
@pytest.fixture() @pytest.fixture()
def backups_backblaze(generic_userdata): def backups_backblaze(generic_userdata):
backups = Backups() Backups.reset()
backups.reset()
return backups
@pytest.fixture() @pytest.fixture()
@ -59,7 +57,7 @@ def dummy_service(tmpdir, backups, raw_dummy_service):
assert not path.exists(repo_path) assert not path.exists(repo_path)
# assert not repo_path # assert not repo_path
backups.init_repo(service) Backups.init_repo(service)
return service return service
@ -83,9 +81,8 @@ def file_backup(tmpdir) -> AbstractBackupProvider:
def test_config_load(generic_userdata): def test_config_load(generic_userdata):
backups = Backups() Backups.reset()
backups.reset() provider = Backups.provider()
provider = backups.provider
assert provider is not None assert provider is not None
assert isinstance(provider, Backblaze) assert isinstance(provider, Backblaze)
@ -114,7 +111,7 @@ def test_backup_simple_file(raw_dummy_service, file_backup):
def test_backup_service(dummy_service, backups): def test_backup_service(dummy_service, backups):
backups.back_up(dummy_service) Backups.back_up(dummy_service)
def test_no_repo(memory_backup): def test_no_repo(memory_backup):
@ -123,9 +120,9 @@ def test_no_repo(memory_backup):
def test_one_snapshot(backups, dummy_service): def test_one_snapshot(backups, dummy_service):
backups.back_up(dummy_service) Backups.back_up(dummy_service)
snaps = backups.get_snapshots(dummy_service) snaps = Backups.get_snapshots(dummy_service)
assert len(snaps) == 1 assert len(snaps) == 1
snap = snaps[0] snap = snaps[0]
assert snap.service_name == dummy_service.get_id() assert snap.service_name == dummy_service.get_id()
@ -137,30 +134,29 @@ def test_restore(backups, dummy_service):
assert file_to_nuke is not None assert file_to_nuke is not None
path_to_nuke = path.join(service_folder, file_to_nuke) path_to_nuke = path.join(service_folder, file_to_nuke)
backups.back_up(dummy_service) Backups.back_up(dummy_service)
snap = backups.get_snapshots(dummy_service)[0] snap = Backups.get_snapshots(dummy_service)[0]
assert snap is not None assert snap is not None
assert path.exists(path_to_nuke) assert path.exists(path_to_nuke)
remove(path_to_nuke) remove(path_to_nuke)
assert not path.exists(path_to_nuke) assert not path.exists(path_to_nuke)
backups.restore_service_from_snapshot(dummy_service, snap.id) Backups.restore_service_from_snapshot(dummy_service, snap.id)
assert path.exists(path_to_nuke) assert path.exists(path_to_nuke)
def test_sizing(backups, dummy_service): def test_sizing(backups, dummy_service):
backups.back_up(dummy_service) Backups.back_up(dummy_service)
snap = backups.get_snapshots(dummy_service)[0] snap = Backups.get_snapshots(dummy_service)[0]
size = backups.service_snapshot_size(dummy_service, snap.id) size = Backups.service_snapshot_size(dummy_service, snap.id)
assert size is not None assert size is not None
assert size > 0 assert size > 0
def test_redis_storage(backups_backblaze): def test_redis_storage(backups_backblaze):
backups = Backups() Backups.reset()
backups.reset() provider = Backups.provider()
provider = backups.provider
assert provider is not None assert provider is not None
@ -168,8 +164,8 @@ def test_redis_storage(backups_backblaze):
assert provider.login == "ID" assert provider.login == "ID"
assert provider.key == "KEY" assert provider.key == "KEY"
backups.store_provider_redis(provider) Backups.store_provider_redis(provider)
restored_provider = backups.load_provider_redis() restored_provider = Backups.load_provider_redis()
assert isinstance(restored_provider, Backblaze) assert isinstance(restored_provider, Backblaze)
assert restored_provider.login == "ID" assert restored_provider.login == "ID"
assert restored_provider.key == "KEY" assert restored_provider.key == "KEY"
@ -177,27 +173,27 @@ def test_redis_storage(backups_backblaze):
# lowlevel # lowlevel
def test_init_tracking_caching(backups, raw_dummy_service): def test_init_tracking_caching(backups, raw_dummy_service):
assert backups._has_redis_init_mark(raw_dummy_service) is False assert Backups._has_redis_init_mark(raw_dummy_service) is False
backups._redis_mark_as_init(raw_dummy_service) Backups._redis_mark_as_init(raw_dummy_service)
assert backups._has_redis_init_mark(raw_dummy_service) is True assert Backups._has_redis_init_mark(raw_dummy_service) is True
assert backups.is_initted(raw_dummy_service) is True assert Backups.is_initted(raw_dummy_service) is True
# lowlevel # lowlevel
def test_init_tracking_caching2(backups, raw_dummy_service): def test_init_tracking_caching2(backups, raw_dummy_service):
assert backups._has_redis_init_mark(raw_dummy_service) is False assert Backups._has_redis_init_mark(raw_dummy_service) is False
backups.init_repo(raw_dummy_service) Backups.init_repo(raw_dummy_service)
assert backups._has_redis_init_mark(raw_dummy_service) is True assert Backups._has_redis_init_mark(raw_dummy_service) is True
# only public API # only public API
def test_init_tracking(backups, raw_dummy_service): def test_init_tracking(backups, raw_dummy_service):
assert backups.is_initted(raw_dummy_service) is False assert Backups.is_initted(raw_dummy_service) is False
backups.init_repo(raw_dummy_service) Backups.init_repo(raw_dummy_service)
assert backups.is_initted(raw_dummy_service) is True assert Backups.is_initted(raw_dummy_service) is True