From b4a3658c78e773fe5df1541c8c2cd2950d35c57c Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Tue, 14 Mar 2023 00:39:15 +0000 Subject: [PATCH] feature(backups): repo init tracking --- selfprivacy_api/backup/__init__.py | 26 ++++++++++++++++ selfprivacy_api/backup/backuper.py | 4 +++ selfprivacy_api/backup/restic_backuper.py | 36 +++++++++++++++++++---- tests/test_graphql/test_backup.py | 28 ++++++++++++++++++ 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index d24872d..550555d 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -17,6 +17,7 @@ from selfprivacy_api.backup.providers import get_provider, get_kind from selfprivacy_api.graphql.queries.providers import BackupProvider REDIS_PROVIDER_KEY = "backups:provider" +REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:" redis = RedisPool().get_connection() @@ -67,6 +68,8 @@ class Backups: @staticmethod def reset(): redis.delete(REDIS_PROVIDER_KEY) + for key in redis.keys(REDIS_INITTED_CACHE_PREFIX + "*"): + redis.delete(key) def lookup_provider(self) -> AbstractBackupProvider: redis_provider = Backups.load_provider_redis() @@ -113,6 +116,29 @@ class Backups: def init_repo(self, service: Service): repo_name = service.get_id() self.provider.backuper.init(repo_name) + self._redis_mark_as_init(service) + + def _has_redis_init_mark(self, service: Service) -> bool: + repo_name = service.get_id() + if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name): + return True + return False + + def _redis_mark_as_init(self, service: Service): + repo_name = service.get_id() + redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1) + + def is_initted(self, service: Service) -> bool: + repo_name = service.get_id() + if self._has_redis_init_mark(service): + return True + + initted = self.provider.backuper.is_initted(repo_name) + if initted: + self._redis_mark_as_init(service) + return True + + return False def get_snapshots(self, service: Service) -> List[Snapshot]: repo_name = service.get_id() diff --git a/selfprivacy_api/backup/backuper.py b/selfprivacy_api/backup/backuper.py index 5d9b1c3..5bba9d5 100644 --- a/selfprivacy_api/backup/backuper.py +++ b/selfprivacy_api/backup/backuper.py @@ -8,6 +8,10 @@ class AbstractBackuper(ABC): def __init__(self): pass + @abstractmethod + def is_initted(self, repo_name: str) -> bool: + raise NotImplementedError + @abstractmethod def start_backup(self, folder: str, repo_name: str): raise NotImplementedError diff --git a/selfprivacy_api/backup/restic_backuper.py b/selfprivacy_api/backup/restic_backuper.py index a4a4830..8d9ac99 100644 --- a/selfprivacy_api/backup/restic_backuper.py +++ b/selfprivacy_api/backup/restic_backuper.py @@ -90,6 +90,20 @@ class ResticBackuper(AbstractBackuper): if not "created restic repository" in output: raise ValueError("cannot init a repo: " + output) + def is_initted(self, repo_name: str) -> bool: + command = self.restic_command( + repo_name, + "check", + "--json", + ) + + with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle: + output = handle.communicate()[0].decode("utf-8") + if not self.has_json(output): + return False + # raise NotImplementedError("error(big): " + output) + return True + def restored_size(self, repo_name, snapshot_id) -> float: """ Size of a snapshot @@ -172,6 +186,16 @@ class ResticBackuper(AbstractBackuper): return snapshots def parse_json_output(self, output: str) -> object: + starting_index = self.json_start(output) + + if starting_index == -1: + raise ValueError( + "There is no json in the restic snapshot output : " + output + ) + + return json.loads(output[starting_index:]) + + def json_start(self, output: str) -> int: indices = [ output.find("["), output.find("{"), @@ -179,10 +203,10 @@ class ResticBackuper(AbstractBackuper): indices = [x for x in indices if x != -1] if indices == []: - raise ValueError( - "There is no json in the restic snapshot output : " + output - ) + return -1 + return min(indices) - starting_index = min(indices) - - return json.loads(output[starting_index:]) + def has_json(self, output: str) -> bool: + if self.json_start(output) == -1: + return False + return True diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index f6f3526..233014f 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -173,3 +173,31 @@ def test_redis_storage(backups_backblaze): assert isinstance(restored_provider, Backblaze) assert restored_provider.login == "ID" assert restored_provider.key == "KEY" + + +# lowlevel +def test_init_tracking_caching(backups, raw_dummy_service): + assert backups._has_redis_init_mark(raw_dummy_service) is False + + backups._redis_mark_as_init(raw_dummy_service) + + assert backups._has_redis_init_mark(raw_dummy_service) is True + assert backups.is_initted(raw_dummy_service) is True + + +# lowlevel +def test_init_tracking_caching2(backups, raw_dummy_service): + assert backups._has_redis_init_mark(raw_dummy_service) is False + + backups.init_repo(raw_dummy_service) + + assert backups._has_redis_init_mark(raw_dummy_service) is True + + +# only public API +def test_init_tracking(backups, raw_dummy_service): + assert backups.is_initted(raw_dummy_service) is False + + backups.init_repo(raw_dummy_service) + + assert backups.is_initted(raw_dummy_service) is True