From 027a37bb47040266edd09d2cab9469dad6ea3a6f Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 21 Aug 2023 11:11:56 +0000 Subject: [PATCH] feature(backup): remember the reason for making a snapshot --- selfprivacy_api/backup/__init__.py | 15 ++-- selfprivacy_api/backup/backuppers/__init__.py | 8 +- .../backup/backuppers/none_backupper.py | 5 +- .../backup/backuppers/restic_backupper.py | 81 ++++++++++--------- selfprivacy_api/backup/tasks.py | 10 ++- .../graphql/common_types/backup.py | 7 ++ selfprivacy_api/models/backup/snapshot.py | 3 + tests/test_graphql/test_backup.py | 7 +- 8 files changed, 85 insertions(+), 51 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index c28c01f..3b141fa 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -23,7 +23,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job from selfprivacy_api.graphql.queries.providers import ( BackupProvider as BackupProviderEnum, ) -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.models.backup.snapshot import Snapshot @@ -264,10 +264,12 @@ class Backups: # Backup @staticmethod - def back_up(service: Service) -> Snapshot: + def back_up( + service: Service, reason: BackupReason = BackupReason.EXPLICIT + ) -> Snapshot: """The top-level function to back up a service""" folders = service.get_folders() - tag = service.get_id() + service_name = service.get_id() job = get_backup_job(service) if job is None: @@ -278,9 +280,10 @@ class Backups: service.pre_backup() snapshot = Backups.provider().backupper.start_backup( folders, - tag, + service_name, + reason=reason, ) - Backups._store_last_snapshot(tag, snapshot) + Backups._store_last_snapshot(service_name, snapshot) service.post_restore() except Exception as error: Jobs.update(job, status=JobStatus.ERROR) @@ -306,7 +309,7 @@ class Backups: snapshot: Snapshot, job: Job, ) -> None: - failsafe_snapshot = Backups.back_up(service) + failsafe_snapshot = Backups.back_up(service, BackupReason.PRE_RESTORE) Jobs.update(job, status=JobStatus.RUNNING) try: diff --git a/selfprivacy_api/backup/backuppers/__init__.py b/selfprivacy_api/backup/backuppers/__init__.py index ccf78b9..0067a41 100644 --- a/selfprivacy_api/backup/backuppers/__init__.py +++ b/selfprivacy_api/backup/backuppers/__init__.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import List from selfprivacy_api.models.backup.snapshot import Snapshot +from selfprivacy_api.graphql.common_types.backup import BackupReason class AbstractBackupper(ABC): @@ -22,7 +23,12 @@ class AbstractBackupper(ABC): raise NotImplementedError @abstractmethod - def start_backup(self, folders: List[str], tag: str) -> Snapshot: + def start_backup( + self, + folders: List[str], + service_name: str, + reason: BackupReason = BackupReason.EXPLICIT, + ) -> Snapshot: """Start a backup of the given folders""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/none_backupper.py b/selfprivacy_api/backup/backuppers/none_backupper.py index 3f9f7fd..429d9ab 100644 --- a/selfprivacy_api/backup/backuppers/none_backupper.py +++ b/selfprivacy_api/backup/backuppers/none_backupper.py @@ -2,6 +2,7 @@ from typing import List from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.backup.backuppers import AbstractBackupper +from selfprivacy_api.graphql.common_types.backup import BackupReason class NoneBackupper(AbstractBackupper): @@ -13,7 +14,9 @@ class NoneBackupper(AbstractBackupper): def set_creds(self, account: str, key: str, repo: str): pass - def start_backup(self, folders: List[str], tag: str): + def start_backup( + self, folders: List[str], tag: str, reason: BackupReason = BackupReason.EXPLICIT + ): raise NotImplementedError def get_snapshots(self) -> List[Snapshot]: diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index 3a5fc49..f5467ff 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -12,6 +12,7 @@ from os.path import exists, join from os import listdir from time import sleep +from selfprivacy_api.graphql.common_types.backup import BackupReason from selfprivacy_api.backup.util import output_yielder, sync from selfprivacy_api.backup.backuppers import AbstractBackupper from selfprivacy_api.models.backup.snapshot import Snapshot @@ -84,7 +85,7 @@ class ResticBackupper(AbstractBackupper): def _password_command(self): return f"echo {LocalBackupSecret.get()}" - def restic_command(self, *args, tag: str = "") -> List[str]: + def restic_command(self, *args, tags: List[str] = []) -> List[str]: command = [ "restic", "-o", @@ -94,13 +95,14 @@ class ResticBackupper(AbstractBackupper): "--password-command", self._password_command(), ] - if tag != "": - command.extend( - [ - "--tag", - tag, - ] - ) + if tags != []: + for tag in tags: + command.extend( + [ + "--tag", + tag, + ] + ) if args: command.extend(ResticBackupper.__flatten_list(args)) return command @@ -164,7 +166,12 @@ class ResticBackupper(AbstractBackupper): return result @unlocked_repo - def start_backup(self, folders: List[str], tag: str) -> Snapshot: + def start_backup( + self, + folders: List[str], + service_name: str, + reason: BackupReason = BackupReason.EXPLICIT, + ) -> Snapshot: """ Start backup with restic """ @@ -173,33 +180,35 @@ class ResticBackupper(AbstractBackupper): # of a string and an array of strings assert not isinstance(folders, str) + tags = [service_name, reason.value] + backup_command = self.restic_command( "backup", "--json", folders, - tag=tag, + tags=tags, ) - messages = [] - - service = get_service_by_id(tag) + service = get_service_by_id(service_name) if service is None: - raise ValueError("No service with id ", tag) - + raise ValueError("No service with id ", service_name) job = get_backup_job(service) + + messages = [] output = [] try: for raw_message in output_yielder(backup_command): output.append(raw_message) - message = self.parse_message( - raw_message, - job, - ) + message = self.parse_message(raw_message, job) messages.append(message) - return ResticBackupper._snapshot_from_backup_messages( - messages, - tag, + id = ResticBackupper._snapshot_id_from_backup_messages(messages) + return Snapshot( + created_at=datetime.datetime.now(datetime.timezone.utc), + id=id, + service_name=service_name, + reason=reason, ) + except ValueError as error: raise ValueError( "Could not create a snapshot: ", @@ -210,13 +219,13 @@ class ResticBackupper(AbstractBackupper): ) from error @staticmethod - def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot: + def _snapshot_id_from_backup_messages(messages) -> Snapshot: for message in messages: if message["message_type"] == "summary": - return ResticBackupper._snapshot_from_fresh_summary( - message, - repo_name, - ) + # There is a discrepancy between versions of restic/rclone + # Some report short_id in this field and some full + return message["snapshot_id"][0:SHORT_ID_LEN] + raise ValueError("no summary message in restic json output") def parse_message(self, raw_message_line: str, job=None) -> dict: @@ -232,16 +241,6 @@ class ResticBackupper(AbstractBackupper): ) return message - @staticmethod - def _snapshot_from_fresh_summary(message: dict, repo_name) -> Snapshot: - return Snapshot( - # There is a discrepancy between versions of restic/rclone - # Some report short_id in this field and some full - id=message["snapshot_id"][0:SHORT_ID_LEN], - created_at=datetime.datetime.now(datetime.timezone.utc), - service_name=repo_name, - ) - def init(self) -> None: init_command = self.restic_command( "init", @@ -475,11 +474,19 @@ class ResticBackupper(AbstractBackupper): def get_snapshots(self) -> List[Snapshot]: """Get all snapshots from the repo""" snapshots = [] + for restic_snapshot in self._load_snapshots(): + # Compatibility with previous snaps: + if len(restic_snapshot["tags"]) == 1: + reason = BackupReason.EXPLICIT + else: + reason = restic_snapshot["tags"][1] + snapshot = Snapshot( id=restic_snapshot["short_id"], created_at=restic_snapshot["time"], service_name=restic_snapshot["tags"][0], + reason=reason, ) snapshots.append(snapshot) diff --git a/selfprivacy_api/backup/tasks.py b/selfprivacy_api/backup/tasks.py index db350d4..546b27c 100644 --- a/selfprivacy_api/backup/tasks.py +++ b/selfprivacy_api/backup/tasks.py @@ -3,7 +3,7 @@ The tasks module contains the worker tasks that are used to back up and restore """ from datetime import datetime, timezone -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.utils.huey import huey @@ -22,11 +22,13 @@ def validate_datetime(dt: datetime) -> bool: # huey tasks need to return something @huey.task() -def start_backup(service: Service) -> bool: +def start_backup( + service: Service, reason: BackupReason = BackupReason.EXPLICIT +) -> bool: """ The worker task that starts the backup process. """ - Backups.back_up(service) + Backups.back_up(service, reason) return True @@ -49,4 +51,4 @@ def automatic_backup(): """ time = datetime.utcnow().replace(tzinfo=timezone.utc) for service in Backups.services_to_back_up(time): - start_backup(service) + start_backup(service, BackupReason.AUTO) diff --git a/selfprivacy_api/graphql/common_types/backup.py b/selfprivacy_api/graphql/common_types/backup.py index 992363b..9eaef12 100644 --- a/selfprivacy_api/graphql/common_types/backup.py +++ b/selfprivacy_api/graphql/common_types/backup.py @@ -8,3 +8,10 @@ from enum import Enum class RestoreStrategy(Enum): INPLACE = "INPLACE" DOWNLOAD_VERIFY_OVERWRITE = "DOWNLOAD_VERIFY_OVERWRITE" + + +@strawberry.enum +class BackupReason(Enum): + EXPLICIT = "EXPLICIT" + AUTO = "AUTO" + PRE_RESTORE = "PRE_RESTORE" diff --git a/selfprivacy_api/models/backup/snapshot.py b/selfprivacy_api/models/backup/snapshot.py index 9893f03..28ad661 100644 --- a/selfprivacy_api/models/backup/snapshot.py +++ b/selfprivacy_api/models/backup/snapshot.py @@ -1,8 +1,11 @@ import datetime from pydantic import BaseModel +from selfprivacy_api.graphql.common_types.backup import BackupReason + class Snapshot(BaseModel): id: str service_name: str created_at: datetime.datetime + reason: BackupReason diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 1990ef7..fdb8497 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -14,7 +14,7 @@ from selfprivacy_api.services import Service, get_all_services from selfprivacy_api.services import get_service_by_id from selfprivacy_api.services.test_service import DummyService from selfprivacy_api.graphql.queries.providers import BackupProvider -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.jobs import Jobs, JobStatus from selfprivacy_api.models.backup.snapshot import Snapshot @@ -428,7 +428,10 @@ def test_forget_snapshot(backups, dummy_service): def test_forget_nonexistent_snapshot(backups, dummy_service): bogus = Snapshot( - id="gibberjibber", service_name="nohoho", created_at=datetime.now(timezone.utc) + id="gibberjibber", + service_name="nohoho", + created_at=datetime.now(timezone.utc), + reason=BackupReason.EXPLICIT, ) with pytest.raises(ValueError): Backups.forget_snapshot(bogus)