refactor(services): make moving a part of generic service functionality
Houkime
parent
b22dfc0469
commit
c947922a5d
|
|
@ -268,6 +268,18 @@ class Jobs:
|
|||
return False
|
||||
|
||||
|
||||
# A terse way to call a common operation, for readability
|
||||
# job.report_progress() would be even better
|
||||
# but it would go against how this file is written
|
||||
def report_progress(progress: int, job: Job, status_text: str) -> None:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text=status_text,
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
|
||||
def _redis_key_from_uuid(uuid_string) -> str:
|
||||
return "jobs:" + str(uuid_string)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,227 +0,0 @@
|
|||
"""Generic handler for moving services"""
|
||||
|
||||
from __future__ import annotations
|
||||
import subprocess
|
||||
import pathlib
|
||||
import shutil
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel
|
||||
from selfprivacy_api.jobs import Job, JobStatus, Jobs
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
from selfprivacy_api.utils.block_devices import BlockDevice
|
||||
from selfprivacy_api.utils import ReadUserData, WriteUserData
|
||||
from selfprivacy_api.services.service import Service
|
||||
from selfprivacy_api.services.owned_path import OwnedPath
|
||||
|
||||
from selfprivacy_api.services.service import StoppedService
|
||||
|
||||
|
||||
class MoveError(Exception):
|
||||
"""Move failed"""
|
||||
|
||||
|
||||
class FolderMoveNames(BaseModel):
|
||||
name: str
|
||||
bind_location: str
|
||||
owner: str
|
||||
group: str
|
||||
|
||||
@staticmethod
|
||||
def from_owned_path(path: OwnedPath) -> FolderMoveNames:
|
||||
return FolderMoveNames(
|
||||
name=FolderMoveNames.get_foldername(path.path),
|
||||
bind_location=path.path,
|
||||
owner=path.owner,
|
||||
group=path.group,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_foldername(path: str) -> str:
|
||||
return path.split("/")[-1]
|
||||
|
||||
@staticmethod
|
||||
def default_foldermoves(service: Service) -> list[FolderMoveNames]:
|
||||
return [
|
||||
FolderMoveNames.from_owned_path(folder)
|
||||
for folder in service.get_owned_folders()
|
||||
]
|
||||
|
||||
|
||||
@huey.task()
|
||||
def move_service(
|
||||
service: Service,
|
||||
new_volume: BlockDevice,
|
||||
job: Job,
|
||||
folder_names: List[FolderMoveNames],
|
||||
userdata_location: str = None, # deprecated, not used
|
||||
):
|
||||
"""
|
||||
Move a service to another volume.
|
||||
Is not allowed to raise errors because it is a task.
|
||||
"""
|
||||
service_name = service.get_display_name()
|
||||
old_volume = service.get_drive()
|
||||
report_progress(0, job, "Performing pre-move checks...")
|
||||
|
||||
try:
|
||||
with ReadUserData() as user_data:
|
||||
if not user_data.get("useBinds", False):
|
||||
raise MoveError("Server is not using binds.")
|
||||
|
||||
check_volume(new_volume, service)
|
||||
check_folders(old_volume, folder_names)
|
||||
|
||||
report_progress(5, job, f"Stopping {service_name}...")
|
||||
|
||||
with StoppedService(service):
|
||||
report_progress(10, job, "Unmounting folders from old volume...")
|
||||
unmount_old_volume(folder_names)
|
||||
|
||||
report_progress(20, job, "Moving data to new volume...")
|
||||
move_folders_to_volume(folder_names, old_volume, new_volume, job)
|
||||
|
||||
report_progress(70, job, f"Making sure {service_name} owns its files...")
|
||||
chown_folders(folder_names, new_volume, job, service)
|
||||
|
||||
report_progress(90, job, f"Mounting {service_name} data...")
|
||||
mount_folders(folder_names, new_volume)
|
||||
|
||||
report_progress(95, job, f"Finishing moving {service_name}...")
|
||||
update_volume_in_userdata(service, new_volume)
|
||||
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
result=f"{service_name} moved successfully.",
|
||||
status_text=f"Starting {service_name}...",
|
||||
progress=100,
|
||||
)
|
||||
except Exception as e:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(e).__name__ + " " + str(e),
|
||||
)
|
||||
|
||||
|
||||
def check_volume(new_volume: BlockDevice, service: Service) -> bool:
|
||||
service_name = service.get_display_name()
|
||||
old_volume_name: str = service.get_drive()
|
||||
|
||||
# Check if we are on the same volume
|
||||
if old_volume_name == new_volume.name:
|
||||
raise MoveError(f"{service_name} is already on volume {new_volume}")
|
||||
|
||||
# Check if there is enough space on the new volume
|
||||
if int(new_volume.fsavail) < service.get_storage_usage():
|
||||
raise MoveError("Not enough space on the new volume.")
|
||||
|
||||
# Make sure the volume is mounted
|
||||
if (
|
||||
not new_volume.is_root()
|
||||
and f"/volumes/{new_volume.name}" not in new_volume.mountpoints
|
||||
):
|
||||
raise MoveError("Volume is not mounted.")
|
||||
|
||||
|
||||
def check_folders(old_volume: BlockDevice, folder_names: List[FolderMoveNames]) -> None:
|
||||
# Make sure current actual directory exists and if its user and group are correct
|
||||
for folder in folder_names:
|
||||
path = pathlib.Path(f"/volumes/{old_volume}/{folder.name}")
|
||||
|
||||
if not path.exists():
|
||||
raise MoveError(f"{path} is not found.")
|
||||
if not path.is_dir():
|
||||
raise MoveError(f"{path} is not a directory.")
|
||||
if path.owner() != folder.owner:
|
||||
raise MoveError(f"{path} owner is not {folder.owner}.")
|
||||
|
||||
|
||||
def unmount_old_volume(folder_names: List[FolderMoveNames]) -> None:
|
||||
for folder in folder_names:
|
||||
try:
|
||||
subprocess.run(
|
||||
["umount", folder.bind_location],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
raise MoveError("Unable to unmount old volume.")
|
||||
|
||||
|
||||
def move_folders_to_volume(
|
||||
folder_names: List[FolderMoveNames],
|
||||
old_volume: BlockDevice,
|
||||
new_volume: BlockDevice,
|
||||
job: Job,
|
||||
) -> None:
|
||||
# Move data to new volume and set correct permissions
|
||||
current_progress = job.progress
|
||||
folder_percentage = 50 // len(folder_names)
|
||||
for folder in folder_names:
|
||||
shutil.move(
|
||||
f"/volumes/{old_volume}/{folder.name}",
|
||||
f"/volumes/{new_volume.name}/{folder.name}",
|
||||
)
|
||||
progress = current_progress + folder_percentage
|
||||
report_progress(progress, job, "Moving data to new volume...")
|
||||
|
||||
|
||||
def chown_folders(
|
||||
folder_names: List[FolderMoveNames], volume: BlockDevice, job: Job, service: Service
|
||||
) -> None:
|
||||
service_name = service.get_display_name()
|
||||
for folder in folder_names:
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"chown",
|
||||
"-R",
|
||||
f"{folder.owner}:{folder.group}",
|
||||
f"/volumes/{volume.name}/{folder.name}",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.output)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
error=f"Unable to set ownership of new volume. {service_name} may not be able to access its files. Continuing anyway.",
|
||||
)
|
||||
|
||||
|
||||
def mount_folders(folder_names: List[FolderMoveNames], volume: BlockDevice) -> None:
|
||||
for folder in folder_names:
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"mount",
|
||||
"--bind",
|
||||
f"/volumes/{volume.name}/{folder.name}",
|
||||
folder.bind_location,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.output)
|
||||
raise MoveError(f"Unable to mount new volume:{error.output}")
|
||||
|
||||
|
||||
def update_volume_in_userdata(service: Service, volume: BlockDevice):
|
||||
with WriteUserData() as user_data:
|
||||
service_id = service.get_id()
|
||||
if "modules" not in user_data:
|
||||
user_data["modules"] = {}
|
||||
if service_id not in user_data["modules"]:
|
||||
user_data["modules"][service_id] = {}
|
||||
user_data["modules"][service_id]["location"] = volume.name
|
||||
|
||||
|
||||
def report_progress(progress: int, job: Job, status_text: str) -> None:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text=status_text,
|
||||
progress=progress,
|
||||
)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
"""Generic handler for moving services"""
|
||||
|
||||
from __future__ import annotations
|
||||
import subprocess
|
||||
import pathlib
|
||||
import shutil
|
||||
from typing import List
|
||||
|
||||
from selfprivacy_api.jobs import Job, report_progress
|
||||
from selfprivacy_api.utils.block_devices import BlockDevice
|
||||
from selfprivacy_api.services.owned_path import OwnedPath
|
||||
|
||||
|
||||
class MoveError(Exception):
|
||||
"""Move failed"""
|
||||
|
||||
def get_foldername(path: str) -> str:
|
||||
return path.split("/")[-1]
|
||||
|
||||
|
||||
|
||||
|
||||
def check_volume(volume: BlockDevice, space_needed: int) -> bool:
|
||||
# Check if there is enough space on the new volume
|
||||
if int(volume.fsavail) < space_needed:
|
||||
raise MoveError("Not enough space on the new volume.")
|
||||
|
||||
# Make sure the volume is mounted
|
||||
if (
|
||||
not volume.is_root()
|
||||
and f"/volumes/{volume.name}" not in volume.mountpoints
|
||||
):
|
||||
raise MoveError("Volume is not mounted.")
|
||||
|
||||
|
||||
def check_folders(current_volume: BlockDevice, folders: List[OwnedPath]) -> None:
|
||||
# Make sure current actual directory exists and if its user and group are correct
|
||||
for folder in folders:
|
||||
path = pathlib.Path(f"/volumes/{current_volume}/{get_foldername(folder)}")
|
||||
|
||||
if not path.exists():
|
||||
raise MoveError(f"{path} is not found.")
|
||||
if not path.is_dir():
|
||||
raise MoveError(f"{path} is not a directory.")
|
||||
if path.owner() != folder.owner:
|
||||
raise MoveError(f"{path} owner is not {folder.owner}.")
|
||||
|
||||
|
||||
def unbind_folders(owned_folders: List[OwnedPath]) -> None:
|
||||
for folder in owned_folders:
|
||||
try:
|
||||
subprocess.run(
|
||||
["umount", folder.path],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
raise MoveError(f"Unable to unmount folder {folder.path}.")
|
||||
|
||||
|
||||
def move_folders_to_volume(
|
||||
folders: List[OwnedPath],
|
||||
old_volume: BlockDevice,
|
||||
new_volume: BlockDevice,
|
||||
job: Job,
|
||||
) -> None:
|
||||
current_progress = job.progress
|
||||
folder_percentage = 50 // len(folders)
|
||||
for folder in folders:
|
||||
folder_name = get_foldername(folder.path)
|
||||
shutil.move(
|
||||
f"/volumes/{old_volume}/{folder_name}",
|
||||
f"/volumes/{new_volume.name}/{folder_name}",
|
||||
)
|
||||
progress = current_progress + folder_percentage
|
||||
report_progress(progress, job, "Moving data to new volume...")
|
||||
|
||||
|
||||
def ensure_folder_ownership(
|
||||
folders: List[OwnedPath], volume: BlockDevice
|
||||
) -> None:
|
||||
for folder in folders:
|
||||
true_location = f"/volumes/{volume.name}/{get_foldername(folder.path)}"
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"chown",
|
||||
"-R",
|
||||
f"{folder.owner}:{folder.group}",
|
||||
# Could we just chown the binded location instead?
|
||||
true_location
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
error_message = f"Unable to set ownership of {true_location} :{error.output}"
|
||||
print(error.output)
|
||||
raise MoveError(error_message)
|
||||
|
||||
|
||||
def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None:
|
||||
for folder in folders:
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"mount",
|
||||
"--bind",
|
||||
f"/volumes/{volume.name}/{get_foldername(folder.path)}",
|
||||
folder.path,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.output)
|
||||
raise MoveError(f"Unable to mount new volume:{error.output}")
|
||||
|
|
@ -4,12 +4,14 @@ from enum import Enum
|
|||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
from selfprivacy_api.jobs import Job
|
||||
from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress
|
||||
|
||||
from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
|
||||
|
||||
from selfprivacy_api.services.generic_size_counter import get_storage_usage
|
||||
from selfprivacy_api.services.owned_path import OwnedPath
|
||||
from selfprivacy_api.services.moving import check_folders, check_volume, unbind_folders, bind_folders, ensure_folder_ownership, MoveError, move_folders_to_volume
|
||||
|
||||
from selfprivacy_api import utils
|
||||
from selfprivacy_api.utils.waitloop import wait_until_true
|
||||
from selfprivacy_api.utils import ReadUserData, WriteUserData
|
||||
|
|
@ -294,6 +296,99 @@ class Service(ABC):
|
|||
def get_foldername(path: str) -> str:
|
||||
return path.split("/")[-1]
|
||||
|
||||
# TODO: with better json utils, it can be one line, and not a separate function
|
||||
@classmethod
|
||||
def set_location(cls, volume: BlockDevice):
|
||||
"""
|
||||
Only changes userdata
|
||||
"""
|
||||
|
||||
with WriteUserData() as user_data:
|
||||
service_id = cls.get_id()
|
||||
if "modules" not in user_data:
|
||||
user_data["modules"] = {}
|
||||
if service_id not in user_data["modules"]:
|
||||
user_data["modules"][service_id] = {}
|
||||
user_data["modules"][service_id]["location"] = volume.name
|
||||
|
||||
def assert_can_move(self, new_volume):
|
||||
"""
|
||||
Checks if the service can be moved to new volume
|
||||
Raises errors if it cannot
|
||||
"""
|
||||
with ReadUserData() as user_data:
|
||||
if not user_data.get("useBinds", False):
|
||||
raise MoveError("Server is not using binds.")
|
||||
|
||||
current_volume_name = self.get_drive()
|
||||
service_name = self.get_display_name()
|
||||
if current_volume_name == new_volume.name:
|
||||
raise MoveError(f"{service_name} is already on volume {new_volume}")
|
||||
|
||||
check_volume(new_volume, space_needed=self.get_storage_usage())
|
||||
|
||||
owned_folders = self.get_owned_folders()
|
||||
if owned_folders == []:
|
||||
raise MoveError("nothing to move")
|
||||
|
||||
check_folders(current_volume_name, owned_folders)
|
||||
|
||||
|
||||
def do_move_to_volume(
|
||||
self,
|
||||
new_volume: BlockDevice,
|
||||
job: Job,
|
||||
):
|
||||
"""
|
||||
Move a service to another volume.
|
||||
Is not allowed to raise errors because it is a task.
|
||||
"""
|
||||
service_name = self.get_display_name()
|
||||
old_volume_name = self.get_drive()
|
||||
owned_folders = self.get_owned_folders()
|
||||
|
||||
report_progress(0, job, "Performing pre-move checks...")
|
||||
|
||||
# TODO: move trying to the task
|
||||
try:
|
||||
report_progress(5, job, f"Stopping {service_name}...")
|
||||
|
||||
with StoppedService(self):
|
||||
report_progress(10, job, "Unmounting folders from old volume...")
|
||||
unbind_folders(owned_folders)
|
||||
|
||||
report_progress(20, job, "Moving data to new volume...")
|
||||
move_folders_to_volume(owned_folders, old_volume_name, new_volume, job)
|
||||
|
||||
report_progress(70, job, f"Making sure {service_name} owns its files...")
|
||||
try:
|
||||
ensure_folder_ownership(owned_folders, new_volume, job, self)
|
||||
except Exception as error:
|
||||
# We have logged it via print and we additionally log it here in the error field
|
||||
# We are continuing anyway but Job has no warning field
|
||||
Jobs.update(job, JobStatus.RUNNING, error=f"Service {service_name} will not be able to write files: " + str(error))
|
||||
|
||||
report_progress(90, job, f"Mounting {service_name} data...")
|
||||
bind_folders(owned_folders, new_volume)
|
||||
|
||||
report_progress(95, job, f"Finishing moving {service_name}...")
|
||||
self.set_location(self, new_volume)
|
||||
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
result=f"{service_name} moved successfully.",
|
||||
status_text=f"Starting {service_name}...",
|
||||
progress=100,
|
||||
)
|
||||
except Exception as e:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(e).__name__ + " " + str(e),
|
||||
)
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def move_to_volume(self, volume: BlockDevice) -> Job:
|
||||
"""Cannot raise errors.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
from selfprivacy_api.services import Service
|
||||
from selfprivacy_api.utils.block_devices import BlockDevice
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
|
||||
|
||||
@huey.task()
|
||||
def move_service(
|
||||
service: Service,
|
||||
new_volume: BlockDevice,
|
||||
):
|
||||
service.move_to_volume(new_volume)
|
||||
Loading…
Reference in New Issue