SelfPrivacy
/
selfprivacy-rest-api
7
2
Fork 1
Code Issues 15 Pull Requests 4 Projects Releases 1 Wiki Activity

feature(backup): remember the reason for making a snapshot

pull/56/head
Houkime 2023-08-21 11:11:56 +00:00
parent 36e915907f
commit 027a37bb47
8 changed files with 85 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
selfprivacy_api/backup/__init__.py
View File

@ -23,7 +23,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job
from selfprivacy_api.graphql.queries.providers import (
BackupProvider as BackupProviderEnum,
)
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason
from selfprivacy_api.models.backup.snapshot import Snapshot
@ -264,10 +264,12 @@ class Backups:
# Backup
@staticmethod
def back_up(service: Service) -> Snapshot:
def back_up(
service: Service, reason: BackupReason = BackupReason.EXPLICIT
) -> Snapshot:
"""The top-level function to back up a service"""
folders = service.get_folders()
tag = service.get_id()
service_name = service.get_id()
job = get_backup_job(service)
if job is None:
@ -278,9 +280,10 @@ class Backups:
service.pre_backup()
snapshot = Backups.provider().backupper.start_backup(
folders,
tag,
service_name,
reason=reason,
)
Backups._store_last_snapshot(tag, snapshot)
Backups._store_last_snapshot(service_name, snapshot)
service.post_restore()
except Exception as error:
Jobs.update(job, status=JobStatus.ERROR)
@ -306,7 +309,7 @@ class Backups:
snapshot: Snapshot,
job: Job,
) -> None:
failsafe_snapshot = Backups.back_up(service)
failsafe_snapshot = Backups.back_up(service, BackupReason.PRE_RESTORE)
Jobs.update(job, status=JobStatus.RUNNING)
try:

8
selfprivacy_api/backup/backuppers/__init__.py
View File

@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
from typing import List
from selfprivacy_api.models.backup.snapshot import Snapshot
from selfprivacy_api.graphql.common_types.backup import BackupReason
class AbstractBackupper(ABC):
@ -22,7 +23,12 @@ class AbstractBackupper(ABC):
raise NotImplementedError
@abstractmethod
def start_backup(self, folders: List[str], tag: str) -> Snapshot:
def start_backup(
self,
folders: List[str],
service_name: str,
reason: BackupReason = BackupReason.EXPLICIT,
) -> Snapshot:
"""Start a backup of the given folders"""
raise NotImplementedError

5
selfprivacy_api/backup/backuppers/none_backupper.py
View File

@ -2,6 +2,7 @@ from typing import List
from selfprivacy_api.models.backup.snapshot import Snapshot
from selfprivacy_api.backup.backuppers import AbstractBackupper
from selfprivacy_api.graphql.common_types.backup import BackupReason
class NoneBackupper(AbstractBackupper):
@ -13,7 +14,9 @@ class NoneBackupper(AbstractBackupper):
def set_creds(self, account: str, key: str, repo: str):
pass
def start_backup(self, folders: List[str], tag: str):
def start_backup(
self, folders: List[str], tag: str, reason: BackupReason = BackupReason.EXPLICIT
):
raise NotImplementedError
def get_snapshots(self) -> List[Snapshot]:

81
selfprivacy_api/backup/backuppers/restic_backupper.py
View File

@ -12,6 +12,7 @@ from os.path import exists, join
from os import listdir
from time import sleep
from selfprivacy_api.graphql.common_types.backup import BackupReason
from selfprivacy_api.backup.util import output_yielder, sync
from selfprivacy_api.backup.backuppers import AbstractBackupper
from selfprivacy_api.models.backup.snapshot import Snapshot
@ -84,7 +85,7 @@ class ResticBackupper(AbstractBackupper):
def _password_command(self):
return f"echo {LocalBackupSecret.get()}"
def restic_command(self, *args, tag: str = "") -> List[str]:
def restic_command(self, *args, tags: List[str] = []) -> List[str]:
command = [
"restic",
"-o",
@ -94,13 +95,14 @@ class ResticBackupper(AbstractBackupper):
"--password-command",
self._password_command(),
]
if tag != "":
command.extend(
[
"--tag",
tag,
]
)
if tags != []:
for tag in tags:
command.extend(
[
"--tag",
tag,
]
)
if args:
command.extend(ResticBackupper.__flatten_list(args))
return command
@ -164,7 +166,12 @@ class ResticBackupper(AbstractBackupper):
return result
@unlocked_repo
def start_backup(self, folders: List[str], tag: str) -> Snapshot:
def start_backup(
self,
folders: List[str],
service_name: str,
reason: BackupReason = BackupReason.EXPLICIT,
) -> Snapshot:
"""
Start backup with restic
"""
@ -173,33 +180,35 @@ class ResticBackupper(AbstractBackupper):
# of a string and an array of strings
assert not isinstance(folders, str)
tags = [service_name, reason.value]
backup_command = self.restic_command(
"backup",
"--json",
folders,
tag=tag,
tags=tags,
)
messages = []
service = get_service_by_id(tag)
service = get_service_by_id(service_name)
if service is None:
raise ValueError("No service with id ", tag)
raise ValueError("No service with id ", service_name)
job = get_backup_job(service)
messages = []
output = []
try:
for raw_message in output_yielder(backup_command):
output.append(raw_message)
message = self.parse_message(
raw_message,
job,
)
message = self.parse_message(raw_message, job)
messages.append(message)
return ResticBackupper._snapshot_from_backup_messages(
messages,
tag,
id = ResticBackupper._snapshot_id_from_backup_messages(messages)
return Snapshot(
created_at=datetime.datetime.now(datetime.timezone.utc),
id=id,
service_name=service_name,
reason=reason,
)
except ValueError as error:
raise ValueError(
"Could not create a snapshot: ",
@ -210,13 +219,13 @@ class ResticBackupper(AbstractBackupper):
) from error
@staticmethod
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:
def _snapshot_id_from_backup_messages(messages) -> Snapshot:
for message in messages:
if message["message_type"] == "summary":
return ResticBackupper._snapshot_from_fresh_summary(
message,
repo_name,
)
# There is a discrepancy between versions of restic/rclone
# Some report short_id in this field and some full
return message["snapshot_id"][0:SHORT_ID_LEN]
raise ValueError("no summary message in restic json output")
def parse_message(self, raw_message_line: str, job=None) -> dict:
@ -232,16 +241,6 @@ class ResticBackupper(AbstractBackupper):
)
return message
@staticmethod
def _snapshot_from_fresh_summary(message: dict, repo_name) -> Snapshot:
return Snapshot(
# There is a discrepancy between versions of restic/rclone
# Some report short_id in this field and some full
id=message["snapshot_id"][0:SHORT_ID_LEN],
created_at=datetime.datetime.now(datetime.timezone.utc),
service_name=repo_name,
)
def init(self) -> None:
init_command = self.restic_command(
"init",
@ -475,11 +474,19 @@ class ResticBackupper(AbstractBackupper):
def get_snapshots(self) -> List[Snapshot]:
"""Get all snapshots from the repo"""
snapshots = []
for restic_snapshot in self._load_snapshots():
# Compatibility with previous snaps:
if len(restic_snapshot["tags"]) == 1:
reason = BackupReason.EXPLICIT
else:
reason = restic_snapshot["tags"][1]
snapshot = Snapshot(
id=restic_snapshot["short_id"],
created_at=restic_snapshot["time"],
service_name=restic_snapshot["tags"][0],
reason=reason,
)
snapshots.append(snapshot)

10
selfprivacy_api/backup/tasks.py
View File

@ -3,7 +3,7 @@ The tasks module contains the worker tasks that are used to back up and restore
"""
from datetime import datetime, timezone
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason
from selfprivacy_api.models.backup.snapshot import Snapshot
from selfprivacy_api.utils.huey import huey
@ -22,11 +22,13 @@ def validate_datetime(dt: datetime) -> bool:
# huey tasks need to return something
@huey.task()
def start_backup(service: Service) -> bool:
def start_backup(
service: Service, reason: BackupReason = BackupReason.EXPLICIT
) -> bool:
"""
The worker task that starts the backup process.
"""
Backups.back_up(service)
Backups.back_up(service, reason)
return True
@ -49,4 +51,4 @@ def automatic_backup():
"""
time = datetime.utcnow().replace(tzinfo=timezone.utc)
for service in Backups.services_to_back_up(time):
start_backup(service)
start_backup(service, BackupReason.AUTO)

7
selfprivacy_api/graphql/common_types/backup.py
View File

@ -8,3 +8,10 @@ from enum import Enum
class RestoreStrategy(Enum):
INPLACE = "INPLACE"
DOWNLOAD_VERIFY_OVERWRITE = "DOWNLOAD_VERIFY_OVERWRITE"
@strawberry.enum
class BackupReason(Enum):
EXPLICIT = "EXPLICIT"
AUTO = "AUTO"
PRE_RESTORE = "PRE_RESTORE"

3
selfprivacy_api/models/backup/snapshot.py
View File

@ -1,8 +1,11 @@
import datetime
from pydantic import BaseModel
from selfprivacy_api.graphql.common_types.backup import BackupReason
class Snapshot(BaseModel):
id: str
service_name: str
created_at: datetime.datetime
reason: BackupReason

7
tests/test_graphql/test_backup.py
View File

@ -14,7 +14,7 @@ from selfprivacy_api.services import Service, get_all_services
from selfprivacy_api.services import get_service_by_id
from selfprivacy_api.services.test_service import DummyService
from selfprivacy_api.graphql.queries.providers import BackupProvider
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy
from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason
from selfprivacy_api.jobs import Jobs, JobStatus
from selfprivacy_api.models.backup.snapshot import Snapshot
@ -428,7 +428,10 @@ def test_forget_snapshot(backups, dummy_service):
def test_forget_nonexistent_snapshot(backups, dummy_service):
bogus = Snapshot(
id="gibberjibber", service_name="nohoho", created_at=datetime.now(timezone.utc)
id="gibberjibber",
service_name="nohoho",
created_at=datetime.now(timezone.utc),
reason=BackupReason.EXPLICIT,
)
with pytest.raises(ValueError):
Backups.forget_snapshot(bogus)