diff --git a/selfprivacy_api/jobs/__init__.py b/selfprivacy_api/jobs/__init__.py index 7310016..7f46e9d 100644 --- a/selfprivacy_api/jobs/__init__.py +++ b/selfprivacy_api/jobs/__init__.py @@ -268,6 +268,18 @@ class Jobs: return False +# A terse way to call a common operation, for readability +# job.report_progress() would be even better +# but it would go against how this file is written +def report_progress(progress: int, job: Job, status_text: str) -> None: + Jobs.update( + job=job, + status=JobStatus.RUNNING, + status_text=status_text, + progress=progress, + ) + + def _redis_key_from_uuid(uuid_string) -> str: return "jobs:" + str(uuid_string) diff --git a/selfprivacy_api/services/generic_service_mover.py b/selfprivacy_api/services/generic_service_mover.py deleted file mode 100644 index 20c717b..0000000 --- a/selfprivacy_api/services/generic_service_mover.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Generic handler for moving services""" - -from __future__ import annotations -import subprocess -import pathlib -import shutil -from typing import List - -from pydantic import BaseModel -from selfprivacy_api.jobs import Job, JobStatus, Jobs -from selfprivacy_api.utils.huey import huey -from selfprivacy_api.utils.block_devices import BlockDevice -from selfprivacy_api.utils import ReadUserData, WriteUserData -from selfprivacy_api.services.service import Service -from selfprivacy_api.services.owned_path import OwnedPath - -from selfprivacy_api.services.service import StoppedService - - -class MoveError(Exception): - """Move failed""" - - -class FolderMoveNames(BaseModel): - name: str - bind_location: str - owner: str - group: str - - @staticmethod - def from_owned_path(path: OwnedPath) -> FolderMoveNames: - return FolderMoveNames( - name=FolderMoveNames.get_foldername(path.path), - bind_location=path.path, - owner=path.owner, - group=path.group, - ) - - @staticmethod - def get_foldername(path: str) -> str: - return path.split("/")[-1] - - @staticmethod - def default_foldermoves(service: Service) -> list[FolderMoveNames]: - return [ - FolderMoveNames.from_owned_path(folder) - for folder in service.get_owned_folders() - ] - - -@huey.task() -def move_service( - service: Service, - new_volume: BlockDevice, - job: Job, - folder_names: List[FolderMoveNames], - userdata_location: str = None, # deprecated, not used -): - """ - Move a service to another volume. - Is not allowed to raise errors because it is a task. - """ - service_name = service.get_display_name() - old_volume = service.get_drive() - report_progress(0, job, "Performing pre-move checks...") - - try: - with ReadUserData() as user_data: - if not user_data.get("useBinds", False): - raise MoveError("Server is not using binds.") - - check_volume(new_volume, service) - check_folders(old_volume, folder_names) - - report_progress(5, job, f"Stopping {service_name}...") - - with StoppedService(service): - report_progress(10, job, "Unmounting folders from old volume...") - unmount_old_volume(folder_names) - - report_progress(20, job, "Moving data to new volume...") - move_folders_to_volume(folder_names, old_volume, new_volume, job) - - report_progress(70, job, f"Making sure {service_name} owns its files...") - chown_folders(folder_names, new_volume, job, service) - - report_progress(90, job, f"Mounting {service_name} data...") - mount_folders(folder_names, new_volume) - - report_progress(95, job, f"Finishing moving {service_name}...") - update_volume_in_userdata(service, new_volume) - - Jobs.update( - job=job, - status=JobStatus.FINISHED, - result=f"{service_name} moved successfully.", - status_text=f"Starting {service_name}...", - progress=100, - ) - except Exception as e: - Jobs.update( - job=job, - status=JobStatus.ERROR, - error=type(e).__name__ + " " + str(e), - ) - - -def check_volume(new_volume: BlockDevice, service: Service) -> bool: - service_name = service.get_display_name() - old_volume_name: str = service.get_drive() - - # Check if we are on the same volume - if old_volume_name == new_volume.name: - raise MoveError(f"{service_name} is already on volume {new_volume}") - - # Check if there is enough space on the new volume - if int(new_volume.fsavail) < service.get_storage_usage(): - raise MoveError("Not enough space on the new volume.") - - # Make sure the volume is mounted - if ( - not new_volume.is_root() - and f"/volumes/{new_volume.name}" not in new_volume.mountpoints - ): - raise MoveError("Volume is not mounted.") - - -def check_folders(old_volume: BlockDevice, folder_names: List[FolderMoveNames]) -> None: - # Make sure current actual directory exists and if its user and group are correct - for folder in folder_names: - path = pathlib.Path(f"/volumes/{old_volume}/{folder.name}") - - if not path.exists(): - raise MoveError(f"{path} is not found.") - if not path.is_dir(): - raise MoveError(f"{path} is not a directory.") - if path.owner() != folder.owner: - raise MoveError(f"{path} owner is not {folder.owner}.") - - -def unmount_old_volume(folder_names: List[FolderMoveNames]) -> None: - for folder in folder_names: - try: - subprocess.run( - ["umount", folder.bind_location], - check=True, - ) - except subprocess.CalledProcessError: - raise MoveError("Unable to unmount old volume.") - - -def move_folders_to_volume( - folder_names: List[FolderMoveNames], - old_volume: BlockDevice, - new_volume: BlockDevice, - job: Job, -) -> None: - # Move data to new volume and set correct permissions - current_progress = job.progress - folder_percentage = 50 // len(folder_names) - for folder in folder_names: - shutil.move( - f"/volumes/{old_volume}/{folder.name}", - f"/volumes/{new_volume.name}/{folder.name}", - ) - progress = current_progress + folder_percentage - report_progress(progress, job, "Moving data to new volume...") - - -def chown_folders( - folder_names: List[FolderMoveNames], volume: BlockDevice, job: Job, service: Service -) -> None: - service_name = service.get_display_name() - for folder in folder_names: - try: - subprocess.run( - [ - "chown", - "-R", - f"{folder.owner}:{folder.group}", - f"/volumes/{volume.name}/{folder.name}", - ], - check=True, - ) - except subprocess.CalledProcessError as error: - print(error.output) - Jobs.update( - job=job, - status=JobStatus.RUNNING, - error=f"Unable to set ownership of new volume. {service_name} may not be able to access its files. Continuing anyway.", - ) - - -def mount_folders(folder_names: List[FolderMoveNames], volume: BlockDevice) -> None: - for folder in folder_names: - try: - subprocess.run( - [ - "mount", - "--bind", - f"/volumes/{volume.name}/{folder.name}", - folder.bind_location, - ], - check=True, - ) - except subprocess.CalledProcessError as error: - print(error.output) - raise MoveError(f"Unable to mount new volume:{error.output}") - - -def update_volume_in_userdata(service: Service, volume: BlockDevice): - with WriteUserData() as user_data: - service_id = service.get_id() - if "modules" not in user_data: - user_data["modules"] = {} - if service_id not in user_data["modules"]: - user_data["modules"][service_id] = {} - user_data["modules"][service_id]["location"] = volume.name - - -def report_progress(progress: int, job: Job, status_text: str) -> None: - Jobs.update( - job=job, - status=JobStatus.RUNNING, - status_text=status_text, - progress=progress, - ) diff --git a/selfprivacy_api/services/moving.py b/selfprivacy_api/services/moving.py new file mode 100644 index 0000000..d667935 --- /dev/null +++ b/selfprivacy_api/services/moving.py @@ -0,0 +1,114 @@ +"""Generic handler for moving services""" + +from __future__ import annotations +import subprocess +import pathlib +import shutil +from typing import List + +from selfprivacy_api.jobs import Job, report_progress +from selfprivacy_api.utils.block_devices import BlockDevice +from selfprivacy_api.services.owned_path import OwnedPath + + +class MoveError(Exception): + """Move failed""" + +def get_foldername(path: str) -> str: + return path.split("/")[-1] + + + + +def check_volume(volume: BlockDevice, space_needed: int) -> bool: + # Check if there is enough space on the new volume + if int(volume.fsavail) < space_needed: + raise MoveError("Not enough space on the new volume.") + + # Make sure the volume is mounted + if ( + not volume.is_root() + and f"/volumes/{volume.name}" not in volume.mountpoints + ): + raise MoveError("Volume is not mounted.") + + +def check_folders(current_volume: BlockDevice, folders: List[OwnedPath]) -> None: + # Make sure current actual directory exists and if its user and group are correct + for folder in folders: + path = pathlib.Path(f"/volumes/{current_volume}/{get_foldername(folder)}") + + if not path.exists(): + raise MoveError(f"{path} is not found.") + if not path.is_dir(): + raise MoveError(f"{path} is not a directory.") + if path.owner() != folder.owner: + raise MoveError(f"{path} owner is not {folder.owner}.") + + +def unbind_folders(owned_folders: List[OwnedPath]) -> None: + for folder in owned_folders: + try: + subprocess.run( + ["umount", folder.path], + check=True, + ) + except subprocess.CalledProcessError: + raise MoveError(f"Unable to unmount folder {folder.path}.") + + +def move_folders_to_volume( + folders: List[OwnedPath], + old_volume: BlockDevice, + new_volume: BlockDevice, + job: Job, +) -> None: + current_progress = job.progress + folder_percentage = 50 // len(folders) + for folder in folders: + folder_name = get_foldername(folder.path) + shutil.move( + f"/volumes/{old_volume}/{folder_name}", + f"/volumes/{new_volume.name}/{folder_name}", + ) + progress = current_progress + folder_percentage + report_progress(progress, job, "Moving data to new volume...") + + +def ensure_folder_ownership( + folders: List[OwnedPath], volume: BlockDevice +) -> None: + for folder in folders: + true_location = f"/volumes/{volume.name}/{get_foldername(folder.path)}" + try: + subprocess.run( + [ + "chown", + "-R", + f"{folder.owner}:{folder.group}", + # Could we just chown the binded location instead? + true_location + ], + check=True, + ) + except subprocess.CalledProcessError as error: + error_message = f"Unable to set ownership of {true_location} :{error.output}" + print(error.output) + raise MoveError(error_message) + + +def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None: + for folder in folders: + try: + subprocess.run( + [ + "mount", + "--bind", + f"/volumes/{volume.name}/{get_foldername(folder.path)}", + folder.path, + ], + check=True, + ) + except subprocess.CalledProcessError as error: + print(error.output) + raise MoveError(f"Unable to mount new volume:{error.output}") diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py index 0cca38a..6255f20 100644 --- a/selfprivacy_api/services/service.py +++ b/selfprivacy_api/services/service.py @@ -4,12 +4,14 @@ from enum import Enum from typing import List, Optional from pydantic import BaseModel -from selfprivacy_api.jobs import Job +from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices from selfprivacy_api.services.generic_size_counter import get_storage_usage from selfprivacy_api.services.owned_path import OwnedPath +from selfprivacy_api.services.moving import check_folders, check_volume, unbind_folders, bind_folders, ensure_folder_ownership, MoveError, move_folders_to_volume + from selfprivacy_api import utils from selfprivacy_api.utils.waitloop import wait_until_true from selfprivacy_api.utils import ReadUserData, WriteUserData @@ -294,6 +296,99 @@ class Service(ABC): def get_foldername(path: str) -> str: return path.split("/")[-1] + # TODO: with better json utils, it can be one line, and not a separate function + @classmethod + def set_location(cls, volume: BlockDevice): + """ + Only changes userdata + """ + + with WriteUserData() as user_data: + service_id = cls.get_id() + if "modules" not in user_data: + user_data["modules"] = {} + if service_id not in user_data["modules"]: + user_data["modules"][service_id] = {} + user_data["modules"][service_id]["location"] = volume.name + + def assert_can_move(self, new_volume): + """ + Checks if the service can be moved to new volume + Raises errors if it cannot + """ + with ReadUserData() as user_data: + if not user_data.get("useBinds", False): + raise MoveError("Server is not using binds.") + + current_volume_name = self.get_drive() + service_name = self.get_display_name() + if current_volume_name == new_volume.name: + raise MoveError(f"{service_name} is already on volume {new_volume}") + + check_volume(new_volume, space_needed=self.get_storage_usage()) + + owned_folders = self.get_owned_folders() + if owned_folders == []: + raise MoveError("nothing to move") + + check_folders(current_volume_name, owned_folders) + + + def do_move_to_volume( + self, + new_volume: BlockDevice, + job: Job, + ): + """ + Move a service to another volume. + Is not allowed to raise errors because it is a task. + """ + service_name = self.get_display_name() + old_volume_name = self.get_drive() + owned_folders = self.get_owned_folders() + + report_progress(0, job, "Performing pre-move checks...") + + # TODO: move trying to the task + try: + report_progress(5, job, f"Stopping {service_name}...") + + with StoppedService(self): + report_progress(10, job, "Unmounting folders from old volume...") + unbind_folders(owned_folders) + + report_progress(20, job, "Moving data to new volume...") + move_folders_to_volume(owned_folders, old_volume_name, new_volume, job) + + report_progress(70, job, f"Making sure {service_name} owns its files...") + try: + ensure_folder_ownership(owned_folders, new_volume, job, self) + except Exception as error: + # We have logged it via print and we additionally log it here in the error field + # We are continuing anyway but Job has no warning field + Jobs.update(job, JobStatus.RUNNING, error=f"Service {service_name} will not be able to write files: " + str(error)) + + report_progress(90, job, f"Mounting {service_name} data...") + bind_folders(owned_folders, new_volume) + + report_progress(95, job, f"Finishing moving {service_name}...") + self.set_location(self, new_volume) + + Jobs.update( + job=job, + status=JobStatus.FINISHED, + result=f"{service_name} moved successfully.", + status_text=f"Starting {service_name}...", + progress=100, + ) + except Exception as e: + Jobs.update( + job=job, + status=JobStatus.ERROR, + error=type(e).__name__ + " " + str(e), + ) + + @abstractmethod def move_to_volume(self, volume: BlockDevice) -> Job: """Cannot raise errors. diff --git a/selfprivacy_api/services/tasks.py b/selfprivacy_api/services/tasks.py new file mode 100644 index 0000000..2cc52ad --- /dev/null +++ b/selfprivacy_api/services/tasks.py @@ -0,0 +1,11 @@ +from selfprivacy_api.services import Service +from selfprivacy_api.utils.block_devices import BlockDevice +from selfprivacy_api.utils.huey import huey + + +@huey.task() +def move_service( + service: Service, + new_volume: BlockDevice, +): + service.move_to_volume(new_volume)