refactor(backups): make backups stateless
Houkime
parent
a2dd47130b
commit
144e4e5e91
|
|
@ -16,6 +16,9 @@ from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
|
|||
from selfprivacy_api.backup.providers import get_provider, get_kind
|
||||
from selfprivacy_api.graphql.queries.providers import BackupProvider
|
||||
|
||||
# a hack to store file path.
|
||||
REDIS_REPO_PATH_KEY = "backups:test_repo_path"
|
||||
|
||||
REDIS_PROVIDER_KEY = "backups:provider"
|
||||
REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:"
|
||||
|
||||
|
|
@ -30,20 +33,30 @@ class Backups:
|
|||
|
||||
provider: AbstractBackupProvider
|
||||
|
||||
def __init__(self, test_repo_file: str = ""):
|
||||
if test_repo_file != "":
|
||||
self.set_localfile_repo(test_repo_file)
|
||||
else:
|
||||
self.provider = self.lookup_provider()
|
||||
|
||||
def set_localfile_repo(self, file_path: str):
|
||||
@staticmethod
|
||||
def set_localfile_repo(file_path: str):
|
||||
ProviderClass = get_provider(BackupProvider.FILE)
|
||||
provider = ProviderClass(file_path)
|
||||
self.provider = provider
|
||||
redis.set(REDIS_REPO_PATH_KEY, file_path)
|
||||
Backups.store_provider_redis(provider)
|
||||
|
||||
@staticmethod
|
||||
def provider():
|
||||
return Backups.lookup_provider()
|
||||
|
||||
@staticmethod
|
||||
def set_provider(kind: str, login: str, key: str):
|
||||
provider = Backups.construct_provider(kind, login, key)
|
||||
Backups.store_provider_redis(provider)
|
||||
|
||||
@staticmethod
|
||||
def construct_provider(kind: str, login: str, key: str):
|
||||
provider_class = get_provider(BackupProvider[kind])
|
||||
|
||||
if kind == "FILE":
|
||||
path = redis.get(REDIS_REPO_PATH_KEY)
|
||||
return provider_class(path)
|
||||
|
||||
return provider_class(login=login, key=key)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -68,19 +81,24 @@ class Backups:
|
|||
@staticmethod
|
||||
def reset():
|
||||
redis.delete(REDIS_PROVIDER_KEY)
|
||||
redis.delete(REDIS_REPO_PATH_KEY)
|
||||
for key in redis.keys(REDIS_INITTED_CACHE_PREFIX + "*"):
|
||||
redis.delete(key)
|
||||
|
||||
def lookup_provider(self) -> AbstractBackupProvider:
|
||||
@staticmethod
|
||||
def lookup_provider() -> AbstractBackupProvider:
|
||||
redis_provider = Backups.load_provider_redis()
|
||||
if redis_provider is not None:
|
||||
return redis_provider
|
||||
|
||||
json_provider = Backups.load_provider_json()
|
||||
if json_provider is not None:
|
||||
Backups.store_provider_redis(json_provider)
|
||||
return json_provider
|
||||
|
||||
return Backups.construct_provider("MEMORY", login="", key="")
|
||||
memory_provider = Backups.construct_provider("MEMORY", login="", key="")
|
||||
Backups.store_provider_redis(memory_provider)
|
||||
return memory_provider
|
||||
|
||||
@staticmethod
|
||||
def load_provider_json() -> AbstractBackupProvider:
|
||||
|
|
@ -105,64 +123,74 @@ class Backups:
|
|||
kind=provider_string, login=account, key=key
|
||||
)
|
||||
|
||||
def back_up(self, service: Service):
|
||||
@staticmethod
|
||||
def back_up(service: Service):
|
||||
folder = service.get_location()
|
||||
repo_name = service.get_id()
|
||||
|
||||
service.pre_backup()
|
||||
self.provider.backuper.start_backup(folder, repo_name)
|
||||
Backups.provider().backuper.start_backup(folder, repo_name)
|
||||
service.post_restore()
|
||||
|
||||
def init_repo(self, service: Service):
|
||||
@staticmethod
|
||||
def init_repo(service: Service):
|
||||
repo_name = service.get_id()
|
||||
self.provider.backuper.init(repo_name)
|
||||
self._redis_mark_as_init(service)
|
||||
Backups.provider().backuper.init(repo_name)
|
||||
Backups._redis_mark_as_init(service)
|
||||
|
||||
def _has_redis_init_mark(self, service: Service) -> bool:
|
||||
@staticmethod
|
||||
def _has_redis_init_mark(service: Service) -> bool:
|
||||
repo_name = service.get_id()
|
||||
if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _redis_mark_as_init(self, service: Service):
|
||||
@staticmethod
|
||||
def _redis_mark_as_init(service: Service):
|
||||
repo_name = service.get_id()
|
||||
redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1)
|
||||
|
||||
def is_initted(self, service: Service) -> bool:
|
||||
@staticmethod
|
||||
def is_initted(service: Service) -> bool:
|
||||
repo_name = service.get_id()
|
||||
if self._has_redis_init_mark(service):
|
||||
if Backups._has_redis_init_mark(service):
|
||||
return True
|
||||
|
||||
initted = self.provider.backuper.is_initted(repo_name)
|
||||
initted = Backups.provider().backuper.is_initted(repo_name)
|
||||
if initted:
|
||||
self._redis_mark_as_init(service)
|
||||
Backups._redis_mark_as_init(service)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def get_snapshots(self, service: Service) -> List[Snapshot]:
|
||||
@staticmethod
|
||||
def get_snapshots(service: Service) -> List[Snapshot]:
|
||||
repo_name = service.get_id()
|
||||
|
||||
return self.provider.backuper.get_snapshots(repo_name)
|
||||
return Backups.provider().backuper.get_snapshots(repo_name)
|
||||
|
||||
def restore_service_from_snapshot(self, service: Service, snapshot_id: str):
|
||||
@staticmethod
|
||||
def restore_service_from_snapshot(service: Service, snapshot_id: str):
|
||||
repo_name = service.get_id()
|
||||
folder = service.get_location()
|
||||
|
||||
self.provider.backuper.restore_from_backup(repo_name, snapshot_id, folder)
|
||||
Backups.provider().backuper.restore_from_backup(repo_name, snapshot_id, folder)
|
||||
|
||||
# Our dummy service is not yet globally registered so this is not testable yet
|
||||
def restore_snapshot(self, snapshot: Snapshot):
|
||||
self.restore_service_from_snapshot(
|
||||
@staticmethod
|
||||
def restore_snapshot(snapshot: Snapshot):
|
||||
Backups.restore_service_from_snapshot(
|
||||
get_service_by_id(snapshot.service_name), snapshot.id
|
||||
)
|
||||
|
||||
def service_snapshot_size(self, service: Service, snapshot_id: str) -> float:
|
||||
@staticmethod
|
||||
def service_snapshot_size(service: Service, snapshot_id: str) -> float:
|
||||
repo_name = service.get_id()
|
||||
return self.provider.backuper.restored_size(repo_name, snapshot_id)
|
||||
return Backups.provider().backuper.restored_size(repo_name, snapshot_id)
|
||||
|
||||
# Our dummy service is not yet globally registered so this is not testable yet
|
||||
def snapshot_restored_size(self, snapshot: Snapshot) -> float:
|
||||
@staticmethod
|
||||
def snapshot_restored_size(snapshot: Snapshot) -> float:
|
||||
return self.service_snapshot_size(
|
||||
get_service_by_id(snapshot.service_name), snapshot.id
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,5 +5,7 @@ from selfprivacy_api.backup.restic_backuper import ResticBackuper
|
|||
class LocalFileBackup(AbstractBackupProvider):
|
||||
backuper = ResticBackuper("", "", "memory")
|
||||
|
||||
def __init__(self, filename: str):
|
||||
# login and key args are for compatibility with generic provider methods. They are ignored.
|
||||
def __init__(self, filename: str, login: str = "", key: str = ""):
|
||||
super().__init__()
|
||||
self.backuper = ResticBackuper("", "", f":local:{filename}/")
|
||||
|
|
|
|||
|
|
@ -12,6 +12,6 @@ class AbstractBackupProvider(ABC):
|
|||
def backuper(self) -> AbstractBackuper:
|
||||
raise NotImplementedError
|
||||
|
||||
def __init__(self, login, key):
|
||||
def __init__(self, login="", key=""):
|
||||
self.login = login
|
||||
self.key = key
|
||||
|
|
|
|||
|
|
@ -22,17 +22,15 @@ REPO_NAME = "test_backup"
|
|||
|
||||
@pytest.fixture(scope="function")
|
||||
def backups(tmpdir):
|
||||
Backups.reset()
|
||||
|
||||
test_repo_path = path.join(tmpdir, "totallyunrelated")
|
||||
backups = Backups(test_repo_path)
|
||||
backups.reset()
|
||||
return backups
|
||||
Backups.set_localfile_repo(test_repo_path)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def backups_backblaze(generic_userdata):
|
||||
backups = Backups()
|
||||
backups.reset()
|
||||
return backups
|
||||
Backups.reset()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
@ -59,7 +57,7 @@ def dummy_service(tmpdir, backups, raw_dummy_service):
|
|||
assert not path.exists(repo_path)
|
||||
# assert not repo_path
|
||||
|
||||
backups.init_repo(service)
|
||||
Backups.init_repo(service)
|
||||
return service
|
||||
|
||||
|
||||
|
|
@ -83,9 +81,8 @@ def file_backup(tmpdir) -> AbstractBackupProvider:
|
|||
|
||||
|
||||
def test_config_load(generic_userdata):
|
||||
backups = Backups()
|
||||
backups.reset()
|
||||
provider = backups.provider
|
||||
Backups.reset()
|
||||
provider = Backups.provider()
|
||||
|
||||
assert provider is not None
|
||||
assert isinstance(provider, Backblaze)
|
||||
|
|
@ -114,7 +111,7 @@ def test_backup_simple_file(raw_dummy_service, file_backup):
|
|||
|
||||
|
||||
def test_backup_service(dummy_service, backups):
|
||||
backups.back_up(dummy_service)
|
||||
Backups.back_up(dummy_service)
|
||||
|
||||
|
||||
def test_no_repo(memory_backup):
|
||||
|
|
@ -123,9 +120,9 @@ def test_no_repo(memory_backup):
|
|||
|
||||
|
||||
def test_one_snapshot(backups, dummy_service):
|
||||
backups.back_up(dummy_service)
|
||||
Backups.back_up(dummy_service)
|
||||
|
||||
snaps = backups.get_snapshots(dummy_service)
|
||||
snaps = Backups.get_snapshots(dummy_service)
|
||||
assert len(snaps) == 1
|
||||
snap = snaps[0]
|
||||
assert snap.service_name == dummy_service.get_id()
|
||||
|
|
@ -137,30 +134,29 @@ def test_restore(backups, dummy_service):
|
|||
assert file_to_nuke is not None
|
||||
path_to_nuke = path.join(service_folder, file_to_nuke)
|
||||
|
||||
backups.back_up(dummy_service)
|
||||
snap = backups.get_snapshots(dummy_service)[0]
|
||||
Backups.back_up(dummy_service)
|
||||
snap = Backups.get_snapshots(dummy_service)[0]
|
||||
assert snap is not None
|
||||
|
||||
assert path.exists(path_to_nuke)
|
||||
remove(path_to_nuke)
|
||||
assert not path.exists(path_to_nuke)
|
||||
|
||||
backups.restore_service_from_snapshot(dummy_service, snap.id)
|
||||
Backups.restore_service_from_snapshot(dummy_service, snap.id)
|
||||
assert path.exists(path_to_nuke)
|
||||
|
||||
|
||||
def test_sizing(backups, dummy_service):
|
||||
backups.back_up(dummy_service)
|
||||
snap = backups.get_snapshots(dummy_service)[0]
|
||||
size = backups.service_snapshot_size(dummy_service, snap.id)
|
||||
Backups.back_up(dummy_service)
|
||||
snap = Backups.get_snapshots(dummy_service)[0]
|
||||
size = Backups.service_snapshot_size(dummy_service, snap.id)
|
||||
assert size is not None
|
||||
assert size > 0
|
||||
|
||||
|
||||
def test_redis_storage(backups_backblaze):
|
||||
backups = Backups()
|
||||
backups.reset()
|
||||
provider = backups.provider
|
||||
Backups.reset()
|
||||
provider = Backups.provider()
|
||||
|
||||
assert provider is not None
|
||||
|
||||
|
|
@ -168,8 +164,8 @@ def test_redis_storage(backups_backblaze):
|
|||
assert provider.login == "ID"
|
||||
assert provider.key == "KEY"
|
||||
|
||||
backups.store_provider_redis(provider)
|
||||
restored_provider = backups.load_provider_redis()
|
||||
Backups.store_provider_redis(provider)
|
||||
restored_provider = Backups.load_provider_redis()
|
||||
assert isinstance(restored_provider, Backblaze)
|
||||
assert restored_provider.login == "ID"
|
||||
assert restored_provider.key == "KEY"
|
||||
|
|
@ -177,27 +173,27 @@ def test_redis_storage(backups_backblaze):
|
|||
|
||||
# lowlevel
|
||||
def test_init_tracking_caching(backups, raw_dummy_service):
|
||||
assert backups._has_redis_init_mark(raw_dummy_service) is False
|
||||
assert Backups._has_redis_init_mark(raw_dummy_service) is False
|
||||
|
||||
backups._redis_mark_as_init(raw_dummy_service)
|
||||
Backups._redis_mark_as_init(raw_dummy_service)
|
||||
|
||||
assert backups._has_redis_init_mark(raw_dummy_service) is True
|
||||
assert backups.is_initted(raw_dummy_service) is True
|
||||
assert Backups._has_redis_init_mark(raw_dummy_service) is True
|
||||
assert Backups.is_initted(raw_dummy_service) is True
|
||||
|
||||
|
||||
# lowlevel
|
||||
def test_init_tracking_caching2(backups, raw_dummy_service):
|
||||
assert backups._has_redis_init_mark(raw_dummy_service) is False
|
||||
assert Backups._has_redis_init_mark(raw_dummy_service) is False
|
||||
|
||||
backups.init_repo(raw_dummy_service)
|
||||
Backups.init_repo(raw_dummy_service)
|
||||
|
||||
assert backups._has_redis_init_mark(raw_dummy_service) is True
|
||||
assert Backups._has_redis_init_mark(raw_dummy_service) is True
|
||||
|
||||
|
||||
# only public API
|
||||
def test_init_tracking(backups, raw_dummy_service):
|
||||
assert backups.is_initted(raw_dummy_service) is False
|
||||
assert Backups.is_initted(raw_dummy_service) is False
|
||||
|
||||
backups.init_repo(raw_dummy_service)
|
||||
Backups.init_repo(raw_dummy_service)
|
||||
|
||||
assert backups.is_initted(raw_dummy_service) is True
|
||||
assert Backups.is_initted(raw_dummy_service) is True
|
||||
|
|
|
|||
Loading…
Reference in New Issue