Merge pull request 'simplify autobackups tasking to avoid deadlocks' (#97) from fix-autobackup-typing into master
continuous-integration/drone/push Build is failing
Details
continuous-integration/drone/push Build is failing
Details
Reviewed-on: #97 Reviewed-by: Inex Code <inex.code@selfprivacy.org>pull/88/head
commit
0e94590420
|
@ -24,6 +24,7 @@
|
||||||
pylsp-mypy
|
pylsp-mypy
|
||||||
python-lsp-black
|
python-lsp-black
|
||||||
python-lsp-server
|
python-lsp-server
|
||||||
|
pyflakes
|
||||||
typer # for strawberry
|
typer # for strawberry
|
||||||
] ++ strawberry-graphql.optional-dependencies.cli));
|
] ++ strawberry-graphql.optional-dependencies.cli));
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,10 @@ def backup_job_type(service: Service) -> str:
|
||||||
return f"{job_type_prefix(service)}.backup"
|
return f"{job_type_prefix(service)}.backup"
|
||||||
|
|
||||||
|
|
||||||
|
def autobackup_job_type() -> str:
|
||||||
|
return "backups.autobackup"
|
||||||
|
|
||||||
|
|
||||||
def restore_job_type(service: Service) -> str:
|
def restore_job_type(service: Service) -> str:
|
||||||
return f"{job_type_prefix(service)}.restore"
|
return f"{job_type_prefix(service)}.restore"
|
||||||
|
|
||||||
|
@ -36,6 +40,17 @@ def is_something_running_for(service: Service) -> bool:
|
||||||
return len(running_jobs) != 0
|
return len(running_jobs) != 0
|
||||||
|
|
||||||
|
|
||||||
|
def add_autobackup_job(services: List[Service]) -> Job:
|
||||||
|
service_names = [s.get_display_name() for s in services]
|
||||||
|
pretty_service_list: str = ", ".join(service_names)
|
||||||
|
job = Jobs.add(
|
||||||
|
type_id=autobackup_job_type(),
|
||||||
|
name="Automatic backup",
|
||||||
|
description=f"Scheduled backup for services: {pretty_service_list}",
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
def add_backup_job(service: Service) -> Job:
|
def add_backup_job(service: Service) -> Job:
|
||||||
if is_something_running_for(service):
|
if is_something_running_for(service):
|
||||||
message = (
|
message = (
|
||||||
|
@ -78,12 +93,14 @@ def get_job_by_type(type_id: str) -> Optional[Job]:
|
||||||
JobStatus.RUNNING,
|
JobStatus.RUNNING,
|
||||||
]:
|
]:
|
||||||
return job
|
return job
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_failed_job_by_type(type_id: str) -> Optional[Job]:
|
def get_failed_job_by_type(type_id: str) -> Optional[Job]:
|
||||||
for job in Jobs.get_jobs():
|
for job in Jobs.get_jobs():
|
||||||
if job.type_id == type_id and job.status == JobStatus.ERROR:
|
if job.type_id == type_id and job.status == JobStatus.ERROR:
|
||||||
return job
|
return job
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_backup_job(service: Service) -> Optional[Job]:
|
def get_backup_job(service: Service) -> Optional[Job]:
|
||||||
|
|
|
@ -12,9 +12,9 @@ from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||||
from selfprivacy_api.utils.huey import huey
|
from selfprivacy_api.utils.huey import huey
|
||||||
from huey import crontab
|
from huey import crontab
|
||||||
|
|
||||||
from selfprivacy_api.services.service import Service
|
|
||||||
from selfprivacy_api.services import get_service_by_id
|
from selfprivacy_api.services import get_service_by_id
|
||||||
from selfprivacy_api.backup import Backups
|
from selfprivacy_api.backup import Backups
|
||||||
|
from selfprivacy_api.backup.jobs import add_autobackup_job
|
||||||
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,26 +72,42 @@ def restore_snapshot(
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def do_autobackup():
|
def do_autobackup() -> None:
|
||||||
"""
|
"""
|
||||||
Body of autobackup task, broken out to test it
|
Body of autobackup task, broken out to test it
|
||||||
For some reason, we cannot launch periodic huey tasks
|
For some reason, we cannot launch periodic huey tasks
|
||||||
inside tests
|
inside tests
|
||||||
"""
|
"""
|
||||||
time = datetime.utcnow().replace(tzinfo=timezone.utc)
|
time = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||||
for service in Backups.services_to_back_up(time):
|
services_to_back_up = Backups.services_to_back_up(time)
|
||||||
handle = start_backup(service.get_id(), BackupReason.AUTO)
|
job = add_autobackup_job(services_to_back_up)
|
||||||
# To be on safe side, we do not do it in parallel
|
|
||||||
handle(blocking=True)
|
progress_per_service = 100 // len(services_to_back_up)
|
||||||
|
progress = 0
|
||||||
|
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||||
|
|
||||||
|
for service in services_to_back_up:
|
||||||
|
try:
|
||||||
|
Backups.back_up(service, BackupReason.AUTO)
|
||||||
|
except Exception as error:
|
||||||
|
Jobs.update(
|
||||||
|
job,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=type(error).__name__ + ": " + str(error),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
progress = progress + progress_per_service
|
||||||
|
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||||
|
|
||||||
|
Jobs.update(job, JobStatus.FINISHED)
|
||||||
|
|
||||||
|
|
||||||
@huey.periodic_task(validate_datetime=validate_datetime)
|
@huey.periodic_task(validate_datetime=validate_datetime)
|
||||||
def automatic_backup() -> bool:
|
def automatic_backup() -> None:
|
||||||
"""
|
"""
|
||||||
The worker periodic task that starts the automatic backup process.
|
The worker periodic task that starts the automatic backup process.
|
||||||
"""
|
"""
|
||||||
do_autobackup()
|
do_autobackup()
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
|
@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
|
||||||
|
|
|
@ -21,9 +21,9 @@ def get_ip4() -> str:
|
||||||
def get_ip6() -> Optional[str]:
|
def get_ip6() -> Optional[str]:
|
||||||
"""Get IPv6 address"""
|
"""Get IPv6 address"""
|
||||||
try:
|
try:
|
||||||
ip6_addresses = subprocess.check_output(["ip", "addr", "show", "dev", "eth0"]).decode(
|
ip6_addresses = subprocess.check_output(
|
||||||
"utf-8"
|
["ip", "addr", "show", "dev", "eth0"]
|
||||||
)
|
).decode("utf-8")
|
||||||
ip6_addresses = re.findall(r"inet6 (\S+)\/\d+", ip6_addresses)
|
ip6_addresses = re.findall(r"inet6 (\S+)\/\d+", ip6_addresses)
|
||||||
for address in ip6_addresses:
|
for address in ip6_addresses:
|
||||||
if ipaddress.IPv6Address(address).is_global:
|
if ipaddress.IPv6Address(address).is_global:
|
||||||
|
|
|
@ -14,11 +14,11 @@ from selfprivacy_api.graphql.common_types.backup import (
|
||||||
from selfprivacy_api.backup import Backups, Snapshot
|
from selfprivacy_api.backup import Backups, Snapshot
|
||||||
from selfprivacy_api.backup.tasks import (
|
from selfprivacy_api.backup.tasks import (
|
||||||
prune_autobackup_snapshots,
|
prune_autobackup_snapshots,
|
||||||
automatic_backup,
|
|
||||||
do_autobackup,
|
do_autobackup,
|
||||||
)
|
)
|
||||||
|
from selfprivacy_api.backup.jobs import autobackup_job_type
|
||||||
|
|
||||||
from tests.test_backup import backups
|
from tests.test_backup import backups, assert_job_finished
|
||||||
from tests.test_graphql.test_services import only_dummy_service
|
from tests.test_graphql.test_services import only_dummy_service
|
||||||
|
|
||||||
|
|
||||||
|
@ -74,6 +74,7 @@ def test_autobackup_taskbody(backups, only_dummy_service):
|
||||||
backup_period = 13 # minutes
|
backup_period = 13 # minutes
|
||||||
|
|
||||||
assert Backups.get_all_snapshots() == []
|
assert Backups.get_all_snapshots() == []
|
||||||
|
assert_job_finished(autobackup_job_type(), count=0)
|
||||||
|
|
||||||
Backups.set_autobackup_period_minutes(backup_period)
|
Backups.set_autobackup_period_minutes(backup_period)
|
||||||
assert Backups.is_time_to_backup_service(dummy_service, now)
|
assert Backups.is_time_to_backup_service(dummy_service, now)
|
||||||
|
@ -88,6 +89,8 @@ def test_autobackup_taskbody(backups, only_dummy_service):
|
||||||
assert snapshots[0].service_name == dummy_service.get_id()
|
assert snapshots[0].service_name == dummy_service.get_id()
|
||||||
assert snapshots[0].reason == BackupReason.AUTO
|
assert snapshots[0].reason == BackupReason.AUTO
|
||||||
|
|
||||||
|
assert_job_finished(autobackup_job_type(), count=1)
|
||||||
|
|
||||||
|
|
||||||
def test_autobackup_timer_periods(backups, dummy_service):
|
def test_autobackup_timer_periods(backups, dummy_service):
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
|
|
Loading…
Reference in New Issue