From 00682723828e04d563e4f6090aebf2df8c234b97 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 26 Feb 2024 15:01:07 +0000 Subject: [PATCH] feature(backups): intermittent commit for binds, to be replaced --- .../graphql/mutations/services_mutations.py | 6 +- selfprivacy_api/services/moving.py | 102 +++++------------- selfprivacy_api/services/owned_path.py | 96 +++++++++++++++++ selfprivacy_api/services/service.py | 33 +++--- selfprivacy_api/utils/block_devices.py | 2 + tests/test_graphql/test_services.py | 6 +- 6 files changed, 154 insertions(+), 91 deletions(-) diff --git a/selfprivacy_api/graphql/mutations/services_mutations.py b/selfprivacy_api/graphql/mutations/services_mutations.py index 911ad26..97eb4d9 100644 --- a/selfprivacy_api/graphql/mutations/services_mutations.py +++ b/selfprivacy_api/graphql/mutations/services_mutations.py @@ -7,6 +7,8 @@ from selfprivacy_api.graphql.common_types.jobs import job_to_api_job from selfprivacy_api.jobs import JobStatus from selfprivacy_api.utils.block_devices import BlockDevices +from traceback import format_tb as format_traceback + from selfprivacy_api.graphql.mutations.mutation_interface import ( GenericJobMutationReturn, GenericMutationReturn, @@ -171,6 +173,7 @@ class ServicesMutations: try: job = move_service(input.service_id, input.location) + except (ServiceNotFoundError, VolumeNotFoundError) as e: return ServiceJobMutationReturn( success=False, @@ -212,4 +215,5 @@ class ServicesMutations: def pretty_error(e: Exception) -> str: - return type(e).__name__ + ": " + str(e) + traceback = "/r".join(format_traceback(e.__traceback__)) + return type(e).__name__ + ": " + str(e) + ": " + traceback diff --git a/selfprivacy_api/services/moving.py b/selfprivacy_api/services/moving.py index ba9b2c9..09af765 100644 --- a/selfprivacy_api/services/moving.py +++ b/selfprivacy_api/services/moving.py @@ -1,26 +1,16 @@ """Generic handler for moving services""" from __future__ import annotations -import subprocess -import pathlib import shutil from typing import List from selfprivacy_api.jobs import Job, report_progress from selfprivacy_api.utils.block_devices import BlockDevice -from selfprivacy_api.services.owned_path import OwnedPath +from selfprivacy_api.services.owned_path import Bind class MoveError(Exception): - """Move failed""" - - -def get_foldername(p: OwnedPath) -> str: - return p.path.split("/")[-1] - - -def location_at_volume(binding_path: OwnedPath, volume_name: str): - return f"/volumes/{volume_name}/{get_foldername(binding_path)}" + """Move of the data has failed""" def check_volume(volume: BlockDevice, space_needed: int) -> None: @@ -33,84 +23,50 @@ def check_volume(volume: BlockDevice, space_needed: int) -> None: raise MoveError("Volume is not mounted.") -def check_folders(volume_name: str, folders: List[OwnedPath]) -> None: +def check_binds(volume_name: str, binds: List[Bind]) -> None: # Make sure current actual directory exists and if its user and group are correct - for folder in folders: - path = pathlib.Path(location_at_volume(folder, volume_name)) - - if not path.exists(): - raise MoveError(f"directory {path} is not found.") - if not path.is_dir(): - raise MoveError(f"{path} is not a directory.") - if path.owner() != folder.owner: - raise MoveError(f"{path} is not owned by {folder.owner}.") + for bind in binds: + bind.validate() -def unbind_folders(owned_folders: List[OwnedPath]) -> None: +def unbind_folders(owned_folders: List[Bind]) -> None: for folder in owned_folders: - try: - subprocess.run( - ["umount", folder.path], - check=True, - ) - except subprocess.CalledProcessError: - raise MoveError(f"Unable to unmount folder {folder.path}.") + folder.unbind() -def move_folders_to_volume( - folders: List[OwnedPath], - old_volume_name: str, # TODO: pass an actual validated block device +# May be moved into Bind +def move_data_to_volume( + binds: List[Bind], new_volume: BlockDevice, job: Job, -) -> None: +) -> List[Bind]: current_progress = job.progress if current_progress is None: current_progress = 0 - progress_per_folder = 50 // len(folders) - for folder in folders: - shutil.move( - location_at_volume(folder, old_volume_name), - location_at_volume(folder, new_volume.name), - ) + progress_per_folder = 50 // len(binds) + for bind in binds: + old_location = bind.location_at_volume() + bind.drive = new_volume + new_location = bind.location_at_volume() + + try: + shutil.move(old_location, new_location) + except Exception as error: + raise MoveError( + f"could not move {old_location} to {new_location} : {str(error)}" + ) from error + progress = current_progress + progress_per_folder report_progress(progress, job, "Moving data to new volume...") + return binds -def ensure_folder_ownership(folders: List[OwnedPath], volume: BlockDevice) -> None: +def ensure_folder_ownership(folders: List[Bind]) -> None: for folder in folders: - true_location = location_at_volume(folder, volume.name) - try: - subprocess.run( - [ - "chown", - "-R", - f"{folder.owner}:{folder.group}", - # Could we just chown the binded location instead? - true_location, - ], - check=True, - ) - except subprocess.CalledProcessError as error: - print(error.output) - error_message = ( - f"Unable to set ownership of {true_location} :{error.output}" - ) - raise MoveError(error_message) + folder.ensure_ownership() -def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None: +def bind_folders(folders: List[Bind], volume: BlockDevice): for folder in folders: - try: - subprocess.run( - [ - "mount", - "--bind", - location_at_volume(folder, volume.name), - folder.path, - ], - check=True, - ) - except subprocess.CalledProcessError as error: - print(error.output) - raise MoveError(f"Unable to mount new volume:{error.output}") + folder.bind() diff --git a/selfprivacy_api/services/owned_path.py b/selfprivacy_api/services/owned_path.py index 23542dc..da40510 100644 --- a/selfprivacy_api/services/owned_path.py +++ b/selfprivacy_api/services/owned_path.py @@ -1,7 +1,103 @@ +from __future__ import annotations +import subprocess +import pathlib from pydantic import BaseModel +from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices + +class BindError(Exception): + pass + + +# May be deprecated because of Binds class OwnedPath(BaseModel): path: str owner: str group: str + + +class Bind: + """ + A directory that resides on some volume but we mount it into fs + where we need it. + Used for service data. + """ + + def __init__(self, binding_path: str, owner: str, group: str, drive: BlockDevice): + self.binding_path = binding_path + self.owner = owner + self.group = group + self.drive = drive + + # TODO: make Service return a list of binds instead of owned paths + @staticmethod + def from_owned_path(path: OwnedPath, drive_name: str) -> Bind: + drive = BlockDevices().get_block_device(drive_name) + if drive is None: + raise BindError(f"No such drive: {drive_name}") + + return Bind( + binding_path=path.path, owner=path.owner, group=path.group, drive=drive + ) + + def bind_foldername(self) -> str: + return self.binding_path.split("/")[-1] + + def location_at_volume(self) -> str: + return f"/volumes/{self.drive.name}/{self.bind_foldername()}" + + def validate(self) -> str: + path = pathlib.Path(self.location_at_volume()) + + if not path.exists(): + raise BindError(f"directory {path} is not found.") + if not path.is_dir(): + raise BindError(f"{path} is not a directory.") + if path.owner() != self.owner: + raise BindError(f"{path} is not owned by {self.owner}.") + + def bind(self) -> None: + try: + subprocess.run( + [ + "mount", + "--bind", + self.location_at_volume(), + self.binding_path, + ], + check=True, + ) + except subprocess.CalledProcessError as error: + print(error.output) + raise BindError(f"Unable to mount new volume:{error.output}") + + def unbind(self) -> None: + try: + subprocess.run( + ["umount", self.binding_path], + check=True, + ) + except subprocess.CalledProcessError: + raise BindError(f"Unable to unmount folder {self.binding_path}.") + pass + + def ensure_ownership(self) -> None: + true_location = self.location_at_volume() + try: + subprocess.run( + [ + "chown", + "-R", + f"{self.owner}:{self.group}", + # Could we just chown the binded location instead? + true_location, + ], + check=True, + ) + except subprocess.CalledProcessError as error: + print(error.output) + error_message = ( + f"Unable to set ownership of {true_location} :{error.output}" + ) + raise BindError(error_message) diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py index e60cf8a..19e395e 100644 --- a/selfprivacy_api/services/service.py +++ b/selfprivacy_api/services/service.py @@ -9,15 +9,15 @@ from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices from selfprivacy_api.services.generic_size_counter import get_storage_usage -from selfprivacy_api.services.owned_path import OwnedPath +from selfprivacy_api.services.owned_path import OwnedPath, Bind from selfprivacy_api.services.moving import ( - check_folders, + check_binds, check_volume, unbind_folders, bind_folders, ensure_folder_ownership, MoveError, - move_folders_to_volume, + move_data_to_volume, ) from selfprivacy_api import utils @@ -319,6 +319,13 @@ class Service(ABC): user_data["modules"][service_id] = {} user_data["modules"][service_id]["location"] = volume.name + def binds(self) -> typing.List[Bind]: + owned_folders = self.get_owned_folders() + + return [ + Bind.from_owned_path(folder, self.get_drive()) for folder in owned_folders + ] + def assert_can_move(self, new_volume): """ Checks if the service can be moved to new volume @@ -338,11 +345,10 @@ class Service(ABC): check_volume(new_volume, space_needed=self.get_storage_usage()) - owned_folders = self.get_owned_folders() - if owned_folders == []: + binds = self.binds() + if binds == []: raise MoveError("nothing to move") - - check_folders(current_volume_name, owned_folders) + check_binds(current_volume_name, binds) def do_move_to_volume( self, @@ -351,21 +357,20 @@ class Service(ABC): ): """ Move a service to another volume. + Note: It may be much simpler to write it per bind, but a bit less safe? """ service_name = self.get_display_name() - # TODO : Make sure device exists - old_volume_name = self.get_drive() - owned_folders = self.get_owned_folders() + binds = self.binds() report_progress(10, job, "Unmounting folders from old volume...") - unbind_folders(owned_folders) + unbind_folders(binds) report_progress(20, job, "Moving data to new volume...") - move_folders_to_volume(owned_folders, old_volume_name, new_volume, job) + binds = move_data_to_volume(binds, new_volume, job) report_progress(70, job, f"Making sure {service_name} owns its files...") try: - ensure_folder_ownership(owned_folders, new_volume) + ensure_folder_ownership(binds) except Exception as error: # We have logged it via print and we additionally log it here in the error field # We are continuing anyway but Job has no warning field @@ -377,7 +382,7 @@ class Service(ABC): ) report_progress(90, job, f"Mounting {service_name} data...") - bind_folders(owned_folders, new_volume) + bind_folders(binds) report_progress(95, job, f"Finishing moving {service_name}...") self.set_location(new_volume) diff --git a/selfprivacy_api/utils/block_devices.py b/selfprivacy_api/utils/block_devices.py index ab3794d..f1a4149 100644 --- a/selfprivacy_api/utils/block_devices.py +++ b/selfprivacy_api/utils/block_devices.py @@ -4,6 +4,8 @@ import subprocess import json import typing +from pydantic import BaseModel + from selfprivacy_api.utils import WriteUserData from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass diff --git a/tests/test_graphql/test_services.py b/tests/test_graphql/test_services.py index 9208371..d509a6f 100644 --- a/tests/test_graphql/test_services.py +++ b/tests/test_graphql/test_services.py @@ -32,9 +32,9 @@ MOVER_MOCK_PROCESS = CompletedProcess(["ls"], returncode=0) @pytest.fixture() -def mock_check_service_mover_folders(mocker): +def mock_check_service_mover_binds(mocker): mock = mocker.patch( - "selfprivacy_api.services.service.check_folders", + "selfprivacy_api.services.service.check_binds", autospec=True, return_value=None, ) @@ -569,7 +569,7 @@ def test_graphql_move_service_without_folders_on_old_volume( def test_graphql_move_service( authorized_client, generic_userdata, - mock_check_service_mover_folders, + mock_check_service_mover_binds, lsblk_singular_mock, dummy_service: DummyService, mock_subprocess_run,