From b01247bc5598eae7206f1cb0db7daa987be52a85 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Thu, 20 Jul 2023 20:11:42 +0300 Subject: [PATCH] refactor: remove legacy backups implementations --- selfprivacy_api/restic_controller/__init__.py | 233 -------- selfprivacy_api/restic_controller/tasks.py | 70 --- .../services/test_restic.py | 506 ------------------ 3 files changed, 809 deletions(-) delete mode 100644 selfprivacy_api/restic_controller/__init__.py delete mode 100644 selfprivacy_api/restic_controller/tasks.py delete mode 100644 tests/test_rest_endpoints/services/test_restic.py diff --git a/selfprivacy_api/restic_controller/__init__.py b/selfprivacy_api/restic_controller/__init__.py deleted file mode 100644 index 4ac84e8..0000000 --- a/selfprivacy_api/restic_controller/__init__.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Restic singleton controller.""" -from datetime import datetime -import json -import subprocess -import os -from enum import Enum -from selfprivacy_api.utils import ReadUserData -from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass - - -class ResticStates(Enum): - """Restic states enum.""" - - NO_KEY = 0 - NOT_INITIALIZED = 1 - INITIALIZED = 2 - BACKING_UP = 3 - RESTORING = 4 - ERROR = 5 - INITIALIZING = 6 - - -class ResticController(metaclass=SingletonMetaclass): - """ - States in wich the restic_controller may be - - no backblaze key - - backblaze key is provided, but repository is not initialized - - backblaze key is provided, repository is initialized - - fetching list of snapshots - - creating snapshot, current progress can be retrieved - - recovering from snapshot - - Any ongoing operation acquires the lock - Current state can be fetched with get_state() - """ - - _initialized = False - - def __init__(self): - if self._initialized: - return - self.state = ResticStates.NO_KEY - self.lock = False - self.progress = 0 - self._backblaze_account = None - self._backblaze_key = None - self._repository_name = None - self.snapshot_list = [] - self.error_message = None - self._initialized = True - self.load_configuration() - self.load_snapshots() - - def load_configuration(self): - """Load current configuration from user data to singleton.""" - with ReadUserData() as user_data: - self._backblaze_account = user_data["backblaze"]["accountId"] - self._backblaze_key = user_data["backblaze"]["accountKey"] - self._repository_name = user_data["backblaze"]["bucket"] - if self._backblaze_account and self._backblaze_key and self._repository_name: - self.state = ResticStates.INITIALIZING - else: - self.state = ResticStates.NO_KEY - - def load_snapshots(self): - """ - Load list of snapshots from repository - """ - backup_listing_command = [ - "restic", - "-o", - self.rclone_args(), - "-r", - self.restic_repo(), - "snapshots", - "--json", - ] - - if self.state in (ResticStates.BACKING_UP, ResticStates.RESTORING): - return - with subprocess.Popen( - backup_listing_command, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as backup_listing_process_descriptor: - snapshots_list = backup_listing_process_descriptor.communicate()[0].decode( - "utf-8" - ) - try: - starting_index = snapshots_list.find("[") - json.loads(snapshots_list[starting_index:]) - self.snapshot_list = json.loads(snapshots_list[starting_index:]) - self.state = ResticStates.INITIALIZED - print(snapshots_list) - except ValueError: - if "Is there a repository at the following location?" in snapshots_list: - self.state = ResticStates.NOT_INITIALIZED - return - self.state = ResticStates.ERROR - self.error_message = snapshots_list - return - - def restic_repo(self): - # https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone - # https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5 - return f"rclone::b2:{self._repository_name}/sfbackup" - - def rclone_args(self): - return "rclone.args=serve restic --stdio" + self.backend_rclone_args() - - def backend_rclone_args(self): - return f"--b2-account {self._backblaze_account} --b2-key {self._backblaze_key}" - - def initialize_repository(self): - """ - Initialize repository with restic - """ - initialize_repository_command = [ - "restic", - "-o", - self.rclone_args(), - "-r", - self.restic_repo(), - "init", - ] - with subprocess.Popen( - initialize_repository_command, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as initialize_repository_process_descriptor: - msg = initialize_repository_process_descriptor.communicate()[0].decode( - "utf-8" - ) - if initialize_repository_process_descriptor.returncode == 0: - self.state = ResticStates.INITIALIZED - else: - self.state = ResticStates.ERROR - self.error_message = msg - - self.state = ResticStates.INITIALIZED - - def start_backup(self): - """ - Start backup with restic - """ - backup_command = [ - "restic", - "-o", - self.rclone_args(), - "-r", - self.restic_repo(), - "--verbose", - "--json", - "backup", - "/var", - ] - with open("/var/backup.log", "w", encoding="utf-8") as log_file: - subprocess.Popen( - backup_command, - shell=False, - stdout=log_file, - stderr=subprocess.STDOUT, - ) - - self.state = ResticStates.BACKING_UP - self.progress = 0 - - def check_progress(self): - """ - Check progress of ongoing backup operation - """ - backup_status_check_command = ["tail", "-1", "/var/backup.log"] - - if self.state in (ResticStates.NO_KEY, ResticStates.NOT_INITIALIZED): - return - - # If the log file does not exists - if os.path.exists("/var/backup.log") is False: - self.state = ResticStates.INITIALIZED - - with subprocess.Popen( - backup_status_check_command, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as backup_status_check_process_descriptor: - backup_process_status = ( - backup_status_check_process_descriptor.communicate()[0].decode("utf-8") - ) - - try: - status = json.loads(backup_process_status) - except ValueError: - print(backup_process_status) - self.error_message = backup_process_status - return - if status["message_type"] == "status": - self.progress = status["percent_done"] - self.state = ResticStates.BACKING_UP - elif status["message_type"] == "summary": - self.state = ResticStates.INITIALIZED - self.progress = 0 - self.snapshot_list.append( - { - "short_id": status["snapshot_id"], - # Current time in format 2021-12-02T00:02:51.086452543+03:00 - "time": datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f%z"), - } - ) - - def restore_from_backup(self, snapshot_id): - """ - Restore from backup with restic - """ - backup_restoration_command = [ - "restic", - "-o", - self.rclone_args(), - "-r", - self.restic_repo(), - "restore", - snapshot_id, - "--target", - "/", - ] - - self.state = ResticStates.RESTORING - - subprocess.run(backup_restoration_command, shell=False) - - self.state = ResticStates.INITIALIZED diff --git a/selfprivacy_api/restic_controller/tasks.py b/selfprivacy_api/restic_controller/tasks.py deleted file mode 100644 index f583d8b..0000000 --- a/selfprivacy_api/restic_controller/tasks.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Tasks for the restic controller.""" -from huey import crontab -from selfprivacy_api.utils.huey import huey -from . import ResticController, ResticStates - - -@huey.task() -def init_restic(): - controller = ResticController() - if controller.state == ResticStates.NOT_INITIALIZED: - initialize_repository() - - -@huey.task() -def update_keys_from_userdata(): - controller = ResticController() - controller.load_configuration() - controller.write_rclone_config() - initialize_repository() - - -# Check every morning at 5:00 AM -@huey.task(crontab(hour=5, minute=0)) -def cron_load_snapshots(): - controller = ResticController() - controller.load_snapshots() - - -# Check every morning at 5:00 AM -@huey.task() -def load_snapshots(): - controller = ResticController() - controller.load_snapshots() - if controller.state == ResticStates.NOT_INITIALIZED: - load_snapshots.schedule(delay=120) - - -@huey.task() -def initialize_repository(): - controller = ResticController() - if controller.state is not ResticStates.NO_KEY: - controller.initialize_repository() - load_snapshots() - - -@huey.task() -def fetch_backup_status(): - controller = ResticController() - if controller.state is ResticStates.BACKING_UP: - controller.check_progress() - if controller.state is ResticStates.BACKING_UP: - fetch_backup_status.schedule(delay=2) - else: - load_snapshots.schedule(delay=240) - - -@huey.task() -def start_backup(): - controller = ResticController() - if controller.state is ResticStates.NOT_INITIALIZED: - resp = initialize_repository() - resp.get() - controller.start_backup() - fetch_backup_status.schedule(delay=3) - - -@huey.task() -def restore_from_backup(snapshot): - controller = ResticController() - controller.restore_from_backup(snapshot) diff --git a/tests/test_rest_endpoints/services/test_restic.py b/tests/test_rest_endpoints/services/test_restic.py deleted file mode 100644 index 844ff34..0000000 --- a/tests/test_rest_endpoints/services/test_restic.py +++ /dev/null @@ -1,506 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -import json -import pytest -from selfprivacy_api.restic_controller import ResticStates - - -def read_json(file_path): - with open(file_path, "r") as f: - return json.load(f) - - -MOCKED_SNAPSHOTS = [ - { - "time": "2021-12-06T09:05:04.224685677+03:00", - "tree": "b76152d1e716d86d420407ead05d9911f2b6d971fe1589c12b63e4de65b14d4e", - "paths": ["/var"], - "hostname": "test-host", - "username": "root", - "id": "f96b428f1ca1252089ea3e25cd8ee33e63fb24615f1cc07559ba907d990d81c5", - "short_id": "f96b428f", - }, - { - "time": "2021-12-08T07:42:06.998894055+03:00", - "parent": "f96b428f1ca1252089ea3e25cd8ee33e63fb24615f1cc07559ba907d990d81c5", - "tree": "8379b4fdc9ee3e9bb7c322f632a7bed9fc334b0258abbf4e7134f8fe5b3d61b0", - "paths": ["/var"], - "hostname": "test-host", - "username": "root", - "id": "db96b36efec97e5ba385099b43f9062d214c7312c20138aee7b8bd2c6cd8995a", - "short_id": "db96b36e", - }, -] - - -class ResticControllerMock: - snapshot_list = MOCKED_SNAPSHOTS - state = ResticStates.INITIALIZED - progress = 0 - error_message = None - - -@pytest.fixture -def mock_restic_controller(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerMock, - ) - return mock - - -class ResticControllerMockNoKey: - snapshot_list = [] - state = ResticStates.NO_KEY - progress = 0 - error_message = None - - -@pytest.fixture -def mock_restic_controller_no_key(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerMockNoKey, - ) - return mock - - -class ResticControllerNotInitialized: - snapshot_list = [] - state = ResticStates.NOT_INITIALIZED - progress = 0 - error_message = None - - -@pytest.fixture -def mock_restic_controller_not_initialized(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerNotInitialized, - ) - return mock - - -class ResticControllerInitializing: - snapshot_list = [] - state = ResticStates.INITIALIZING - progress = 0 - error_message = None - - -@pytest.fixture -def mock_restic_controller_initializing(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerInitializing, - ) - return mock - - -class ResticControllerBackingUp: - snapshot_list = MOCKED_SNAPSHOTS - state = ResticStates.BACKING_UP - progress = 0.42 - error_message = None - - -@pytest.fixture -def mock_restic_controller_backing_up(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerBackingUp, - ) - return mock - - -class ResticControllerError: - snapshot_list = MOCKED_SNAPSHOTS - state = ResticStates.ERROR - progress = 0 - error_message = "Error message" - - -@pytest.fixture -def mock_restic_controller_error(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerError, - ) - return mock - - -class ResticControllerRestoring: - snapshot_list = MOCKED_SNAPSHOTS - state = ResticStates.RESTORING - progress = 0 - error_message = None - - -@pytest.fixture -def mock_restic_controller_restoring(mocker): - mock = mocker.patch( - "selfprivacy_api.rest.services.ResticController", - autospec=True, - return_value=ResticControllerRestoring, - ) - return mock - - -@pytest.fixture -def mock_restic_tasks(mocker): - mock = mocker.patch("selfprivacy_api.rest.services.restic_tasks", autospec=True) - return mock - - -@pytest.fixture -def undefined_settings(mocker, datadir): - mocker.patch("selfprivacy_api.utils.USERDATA_FILE", new=datadir / "undefined.json") - assert "backup" not in read_json(datadir / "undefined.json") - return datadir - - -@pytest.fixture -def some_settings(mocker, datadir): - mocker.patch( - "selfprivacy_api.utils.USERDATA_FILE", new=datadir / "some_values.json" - ) - assert "backup" in read_json(datadir / "some_values.json") - assert read_json(datadir / "some_values.json")["backup"]["provider"] == "BACKBLAZE" - assert read_json(datadir / "some_values.json")["backup"]["accountId"] == "ID" - assert read_json(datadir / "some_values.json")["backup"]["accountKey"] == "KEY" - assert read_json(datadir / "some_values.json")["backup"]["bucket"] == "BUCKET" - return datadir - - -@pytest.fixture -def no_values(mocker, datadir): - mocker.patch("selfprivacy_api.utils.USERDATA_FILE", new=datadir / "no_values.json") - assert "backup" in read_json(datadir / "no_values.json") - assert "provider" not in read_json(datadir / "no_values.json")["backup"] - assert "accountId" not in read_json(datadir / "no_values.json")["backup"] - assert "accountKey" not in read_json(datadir / "no_values.json")["backup"] - assert "bucket" not in read_json(datadir / "no_values.json")["backup"] - return datadir - - -def test_get_snapshots_unauthorized(client, mock_restic_controller, mock_restic_tasks): - response = client.get("/services/restic/backup/list") - assert response.status_code == 401 - - -def test_get_snapshots(authorized_client, mock_restic_controller, mock_restic_tasks): - response = authorized_client.get("/services/restic/backup/list") - assert response.status_code == 200 - assert response.json() == MOCKED_SNAPSHOTS - - -def test_create_backup_unauthorized(client, mock_restic_controller, mock_restic_tasks): - response = client.put("/services/restic/backup/create") - assert response.status_code == 401 - - -def test_create_backup(authorized_client, mock_restic_controller, mock_restic_tasks): - response = authorized_client.put("/services/restic/backup/create") - assert response.status_code == 200 - assert mock_restic_tasks.start_backup.call_count == 1 - - -def test_create_backup_without_key( - authorized_client, mock_restic_controller_no_key, mock_restic_tasks -): - response = authorized_client.put("/services/restic/backup/create") - assert response.status_code == 400 - assert mock_restic_tasks.start_backup.call_count == 0 - - -def test_create_backup_initializing( - authorized_client, mock_restic_controller_initializing, mock_restic_tasks -): - response = authorized_client.put("/services/restic/backup/create") - assert response.status_code == 400 - assert mock_restic_tasks.start_backup.call_count == 0 - - -def test_create_backup_backing_up( - authorized_client, mock_restic_controller_backing_up, mock_restic_tasks -): - response = authorized_client.put("/services/restic/backup/create") - assert response.status_code == 409 - assert mock_restic_tasks.start_backup.call_count == 0 - - -def test_check_backup_status_unauthorized( - client, mock_restic_controller, mock_restic_tasks -): - response = client.get("/services/restic/backup/status") - assert response.status_code == 401 - - -def test_check_backup_status( - authorized_client, mock_restic_controller, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "INITIALIZED", - "progress": 0, - "error_message": None, - } - - -def test_check_backup_status_no_key( - authorized_client, mock_restic_controller_no_key, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "NO_KEY", - "progress": 0, - "error_message": None, - } - - -def test_check_backup_status_not_initialized( - authorized_client, mock_restic_controller_not_initialized, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "NOT_INITIALIZED", - "progress": 0, - "error_message": None, - } - - -def test_check_backup_status_initializing( - authorized_client, mock_restic_controller_initializing, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "INITIALIZING", - "progress": 0, - "error_message": None, - } - - -def test_check_backup_status_backing_up( - authorized_client, mock_restic_controller_backing_up -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "BACKING_UP", - "progress": 0.42, - "error_message": None, - } - - -def test_check_backup_status_error( - authorized_client, mock_restic_controller_error, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "ERROR", - "progress": 0, - "error_message": "Error message", - } - - -def test_check_backup_status_restoring( - authorized_client, mock_restic_controller_restoring, mock_restic_tasks -): - response = authorized_client.get("/services/restic/backup/status") - assert response.status_code == 200 - assert response.json() == { - "status": "RESTORING", - "progress": 0, - "error_message": None, - } - - -def test_reload_unauthenticated(client, mock_restic_controller, mock_restic_tasks): - response = client.get("/services/restic/backup/reload") - assert response.status_code == 401 - - -def test_backup_reload(authorized_client, mock_restic_controller, mock_restic_tasks): - response = authorized_client.get("/services/restic/backup/reload") - assert response.status_code == 200 - assert mock_restic_tasks.load_snapshots.call_count == 1 - - -def test_backup_restore_unauthorized(client, mock_restic_controller, mock_restic_tasks): - response = client.put("/services/restic/backup/restore") - assert response.status_code == 401 - - -def test_backup_restore_without_backup_id( - authorized_client, mock_restic_controller, mock_restic_tasks -): - response = authorized_client.put("/services/restic/backup/restore", json={}) - assert response.status_code == 422 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_with_nonexistent_backup_id( - authorized_client, mock_restic_controller, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "nonexistent"} - ) - assert response.status_code == 404 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_no_key( - authorized_client, mock_restic_controller_no_key, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 400 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_not_initialized( - authorized_client, mock_restic_controller_not_initialized, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 400 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_initializing( - authorized_client, mock_restic_controller_initializing, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 400 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_backing_up( - authorized_client, mock_restic_controller_backing_up, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 409 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_restoring( - authorized_client, mock_restic_controller_restoring, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 409 - assert mock_restic_tasks.restore_from_backup.call_count == 0 - - -def test_backup_restore_when_error( - authorized_client, mock_restic_controller_error, mock_restic_tasks -): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 200 - assert mock_restic_tasks.restore_from_backup.call_count == 1 - - -def test_backup_restore(authorized_client, mock_restic_controller, mock_restic_tasks): - response = authorized_client.put( - "/services/restic/backup/restore", json={"backupId": "f96b428f"} - ) - assert response.status_code == 200 - assert mock_restic_tasks.restore_from_backup.call_count == 1 - - -def test_set_backblaze_config_unauthorized( - client, mock_restic_controller, mock_restic_tasks, some_settings -): - response = client.put("/services/restic/backblaze/config") - assert response.status_code == 401 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 0 - - -def test_set_backblaze_config_without_arguments( - authorized_client, mock_restic_controller, mock_restic_tasks, some_settings -): - response = authorized_client.put("/services/restic/backblaze/config") - assert response.status_code == 422 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 0 - - -def test_set_backblaze_config_without_all_values( - authorized_client, mock_restic_controller, mock_restic_tasks, some_settings -): - response = authorized_client.put( - "/services/restic/backblaze/config", - json={"accountId": "123", "applicationKey": "456"}, - ) - assert response.status_code == 422 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 0 - - -def test_set_backblaze_config( - authorized_client, mock_restic_controller, mock_restic_tasks, some_settings -): - response = authorized_client.put( - "/services/restic/backblaze/config", - json={"accountId": "123", "accountKey": "456", "bucket": "789"}, - ) - assert response.status_code == 200 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 1 - assert read_json(some_settings / "some_values.json")["backup"] == { - "provider": "BACKBLAZE", - "accountId": "123", - "accountKey": "456", - "bucket": "789", - } - - -def test_set_backblaze_config_on_undefined( - authorized_client, mock_restic_controller, mock_restic_tasks, undefined_settings -): - response = authorized_client.put( - "/services/restic/backblaze/config", - json={"accountId": "123", "accountKey": "456", "bucket": "789"}, - ) - assert response.status_code == 200 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 1 - assert read_json(undefined_settings / "undefined.json")["backup"] == { - "provider": "BACKBLAZE", - "accountId": "123", - "accountKey": "456", - "bucket": "789", - } - - -def test_set_backblaze_config_on_no_values( - authorized_client, mock_restic_controller, mock_restic_tasks, no_values -): - response = authorized_client.put( - "/services/restic/backblaze/config", - json={"accountId": "123", "accountKey": "456", "bucket": "789"}, - ) - assert response.status_code == 200 - assert mock_restic_tasks.update_keys_from_userdata.call_count == 1 - assert read_json(no_values / "no_values.json")["backup"] == { - "provider": "BACKBLAZE", - "accountId": "123", - "accountKey": "456", - "bucket": "789", - }