Compare commits
19 Commits
censor-err
...
master
Author | SHA1 | Date |
---|---|---|
|
a742e66cc3 | |
|
4f1d44ce74 | |
![]() |
8e8e76a954 | |
|
5a100ec33a | |
|
524adaa8bc | |
|
5e93e6499f | |
|
3302fe2818 | |
![]() |
9ee72c1fcb | |
|
5ec677339b | |
![]() |
f2446dcee2 | |
![]() |
97960f77f2 | |
![]() |
677ed27773 | |
![]() |
6f38b2309f | |
![]() |
baf7843349 | |
![]() |
8e48a5ad5f | |
![]() |
fde461b4b9 | |
![]() |
9954737791 | |
![]() |
2b19633cbd | |
![]() |
83592b7bf4 |
14
README.md
14
README.md
|
@ -1,6 +1,8 @@
|
||||||
# SelfPrivacy GraphQL API which allows app to control your server
|
# SelfPrivacy GraphQL API which allows app to control your server
|
||||||
|
|
||||||
## build
|
![CI status](https://ci.selfprivacy.org/api/badges/SelfPrivacy/selfprivacy-rest-api/status.svg)
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ nix build
|
$ nix build
|
||||||
|
@ -8,7 +10,7 @@ $ nix build
|
||||||
|
|
||||||
In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
|
In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
|
||||||
|
|
||||||
## develop
|
## Develop
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ nix develop
|
$ nix develop
|
||||||
|
@ -21,10 +23,10 @@ Type "help", "copyright", "credits" or "license" for more information.
|
||||||
If you don't have experimental flakes enabled, you can use the following command:
|
If you don't have experimental flakes enabled, you can use the following command:
|
||||||
|
|
||||||
```console
|
```console
|
||||||
nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
|
$ nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
|
||||||
```
|
```
|
||||||
|
|
||||||
## testing
|
## Testing
|
||||||
|
|
||||||
Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
|
Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
|
||||||
```console
|
```console
|
||||||
|
@ -61,7 +63,7 @@ $ TMPDIR=".nixos-vm-tmp-dir" nix run .#checks.x86_64-linux.default.driverInterac
|
||||||
|
|
||||||
Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.
|
Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.
|
||||||
|
|
||||||
## dependencies and dependant modules
|
## Dependencies and Dependant Modules
|
||||||
|
|
||||||
This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.
|
This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.
|
||||||
|
|
||||||
|
@ -85,6 +87,6 @@ $ nix flake metadata git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nix
|
||||||
|
|
||||||
Nix code for NixOS service module for API is located in NixOS configuration repository.
|
Nix code for NixOS service module for API is located in NixOS configuration repository.
|
||||||
|
|
||||||
## troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.
|
Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.
|
||||||
|
|
|
@ -27,6 +27,7 @@
|
||||||
python-lsp-server
|
python-lsp-server
|
||||||
pyflakes
|
pyflakes
|
||||||
typer # for strawberry
|
typer # for strawberry
|
||||||
|
types-redis # for mypy
|
||||||
] ++ strawberry-graphql.optional-dependencies.cli));
|
] ++ strawberry-graphql.optional-dependencies.cli));
|
||||||
|
|
||||||
vmtest-src-dir = "/root/source";
|
vmtest-src-dir = "/root/source";
|
||||||
|
|
|
@ -27,4 +27,4 @@ async def get_token_header(
|
||||||
|
|
||||||
def get_api_version() -> str:
|
def get_api_version() -> str:
|
||||||
"""Get API version"""
|
"""Get API version"""
|
||||||
return "3.1.0"
|
return "3.2.1"
|
||||||
|
|
|
@ -8,9 +8,12 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||||
GenericJobMutationReturn,
|
GenericJobMutationReturn,
|
||||||
GenericMutationReturn,
|
GenericMutationReturn,
|
||||||
MutationReturnInterface,
|
MutationReturnInterface,
|
||||||
|
GenericJobMutationReturn,
|
||||||
)
|
)
|
||||||
|
|
||||||
import selfprivacy_api.actions.system as system_actions
|
import selfprivacy_api.actions.system as system_actions
|
||||||
|
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||||
|
from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
|
||||||
import selfprivacy_api.actions.ssh as ssh_actions
|
import selfprivacy_api.actions.ssh as ssh_actions
|
||||||
|
|
||||||
|
|
||||||
|
@ -195,3 +198,14 @@ class SystemMutations:
|
||||||
message=f"Failed to pull repository changes:\n{result.data}",
|
message=f"Failed to pull repository changes:\n{result.data}",
|
||||||
code=500,
|
code=500,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||||
|
def nix_collect_garbage(self) -> GenericJobMutationReturn:
|
||||||
|
job = start_nix_collect_garbage()
|
||||||
|
|
||||||
|
return GenericJobMutationReturn(
|
||||||
|
success=True,
|
||||||
|
code=200,
|
||||||
|
message="Garbage collector started...",
|
||||||
|
job=job_to_api_job(job),
|
||||||
|
)
|
||||||
|
|
|
@ -34,6 +34,24 @@ class BackupConfiguration:
|
||||||
location_id: typing.Optional[str]
|
location_id: typing.Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: Ideally this should not be done in API but making an internal Service requires more work
|
||||||
|
# than to make an API record about a service
|
||||||
|
def tombstone_service(service_id: str) -> Service:
|
||||||
|
return Service(
|
||||||
|
id=service_id,
|
||||||
|
display_name=f"{service_id} (Orphaned)",
|
||||||
|
description="",
|
||||||
|
svg_icon="",
|
||||||
|
is_movable=False,
|
||||||
|
is_required=False,
|
||||||
|
is_enabled=False,
|
||||||
|
status=ServiceStatusEnum.OFF,
|
||||||
|
url=None,
|
||||||
|
can_be_backed_up=False,
|
||||||
|
backup_description="",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@strawberry.type
|
@strawberry.type
|
||||||
class Backup:
|
class Backup:
|
||||||
@strawberry.field
|
@strawberry.field
|
||||||
|
@ -55,27 +73,21 @@ class Backup:
|
||||||
result = []
|
result = []
|
||||||
snapshots = Backups.get_all_snapshots()
|
snapshots = Backups.get_all_snapshots()
|
||||||
for snap in snapshots:
|
for snap in snapshots:
|
||||||
|
api_service = None
|
||||||
service = get_service_by_id(snap.service_name)
|
service = get_service_by_id(snap.service_name)
|
||||||
|
|
||||||
if service is None:
|
if service is None:
|
||||||
service = Service(
|
api_service = tombstone_service(snap.service_name)
|
||||||
id=snap.service_name,
|
|
||||||
display_name=f"{snap.service_name} (Orphaned)",
|
|
||||||
description="",
|
|
||||||
svg_icon="",
|
|
||||||
is_movable=False,
|
|
||||||
is_required=False,
|
|
||||||
is_enabled=False,
|
|
||||||
status=ServiceStatusEnum.OFF,
|
|
||||||
url=None,
|
|
||||||
dns_records=None,
|
|
||||||
can_be_backed_up=False,
|
|
||||||
backup_description="",
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
service = service_to_graphql_service(service)
|
api_service = service_to_graphql_service(service)
|
||||||
|
if api_service is None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"Could not construct API Service record for:{snap.service_name}. This should be unreachable and is a bug if you see it."
|
||||||
|
)
|
||||||
|
|
||||||
graphql_snap = SnapshotInfo(
|
graphql_snap = SnapshotInfo(
|
||||||
id=snap.id,
|
id=snap.id,
|
||||||
service=service,
|
service=api_service,
|
||||||
created_at=snap.created_at,
|
created_at=snap.created_at,
|
||||||
reason=snap.reason,
|
reason=snap.reason,
|
||||||
)
|
)
|
||||||
|
|
|
@ -14,6 +14,7 @@ class DnsProvider(Enum):
|
||||||
class ServerProvider(Enum):
|
class ServerProvider(Enum):
|
||||||
HETZNER = "HETZNER"
|
HETZNER = "HETZNER"
|
||||||
DIGITALOCEAN = "DIGITALOCEAN"
|
DIGITALOCEAN = "DIGITALOCEAN"
|
||||||
|
OTHER = "OTHER"
|
||||||
|
|
||||||
|
|
||||||
@strawberry.enum
|
@strawberry.enum
|
||||||
|
|
|
@ -67,8 +67,8 @@ def move_folder(
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
|
data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
except Exception as e:
|
except Exception as error:
|
||||||
print(f"Error creating data path: {e}")
|
print(f"Error creating data path: {error}")
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -0,0 +1,147 @@
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from typing import Tuple, Iterable
|
||||||
|
|
||||||
|
from selfprivacy_api.utils.huey import huey
|
||||||
|
|
||||||
|
from selfprivacy_api.jobs import JobStatus, Jobs, Job
|
||||||
|
|
||||||
|
|
||||||
|
class ShellException(Exception):
|
||||||
|
"""Shell-related errors"""
|
||||||
|
|
||||||
|
|
||||||
|
COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
|
||||||
|
RESULT_WAS_NOT_FOUND_ERROR = (
|
||||||
|
"We are sorry, garbage collection result was not found. "
|
||||||
|
"Something went wrong, please report this to the support chat."
|
||||||
|
)
|
||||||
|
CLEAR_COMPLETED = "Garbage collection completed."
|
||||||
|
|
||||||
|
|
||||||
|
def delete_old_gens_and_return_dead_report() -> str:
|
||||||
|
subprocess.run(
|
||||||
|
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
|
||||||
|
"utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
return " " if result is None else result
|
||||||
|
|
||||||
|
|
||||||
|
def run_nix_collect_garbage() -> Iterable[bytes]:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||||
|
)
|
||||||
|
return process.stdout if process.stdout else iter([])
|
||||||
|
|
||||||
|
|
||||||
|
def parse_line(job: Job, line: str) -> Job:
|
||||||
|
"""
|
||||||
|
We parse the string for the presence of a final line,
|
||||||
|
with the final amount of space cleared.
|
||||||
|
Simply put, we're just looking for a similar string:
|
||||||
|
"1537 store paths deleted, 339.84 MiB freed".
|
||||||
|
"""
|
||||||
|
pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
|
||||||
|
match = re.search(pattern, line)
|
||||||
|
|
||||||
|
if match is None:
|
||||||
|
raise ShellException("nix returned gibberish output")
|
||||||
|
|
||||||
|
else:
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.FINISHED,
|
||||||
|
status_text=CLEAR_COMPLETED,
|
||||||
|
result=f"{match.group(0)} have been cleared",
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
|
||||||
|
completed_packages = 0
|
||||||
|
prev_progress = 0
|
||||||
|
|
||||||
|
for line in stream:
|
||||||
|
line = line.decode("utf-8")
|
||||||
|
|
||||||
|
if "deleting '/nix/store/" in line:
|
||||||
|
completed_packages += 1
|
||||||
|
percent = int((completed_packages / total_dead_packages) * 100)
|
||||||
|
|
||||||
|
if percent - prev_progress >= 5:
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.RUNNING,
|
||||||
|
progress=percent,
|
||||||
|
status_text="Cleaning...",
|
||||||
|
)
|
||||||
|
prev_progress = percent
|
||||||
|
|
||||||
|
elif "store paths deleted," in line:
|
||||||
|
parse_line(job, line)
|
||||||
|
|
||||||
|
|
||||||
|
def get_dead_packages(output) -> Tuple[int, float]:
|
||||||
|
dead = len(re.findall("/nix/store/", output))
|
||||||
|
percent = 0
|
||||||
|
if dead != 0:
|
||||||
|
percent = 100 / dead
|
||||||
|
return dead, percent
|
||||||
|
|
||||||
|
|
||||||
|
@huey.task()
|
||||||
|
def calculate_and_clear_dead_paths(job: Job):
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.RUNNING,
|
||||||
|
progress=0,
|
||||||
|
status_text="Calculate the number of dead packages...",
|
||||||
|
)
|
||||||
|
|
||||||
|
dead_packages, package_equal_to_percent = get_dead_packages(
|
||||||
|
delete_old_gens_and_return_dead_report()
|
||||||
|
)
|
||||||
|
|
||||||
|
if dead_packages == 0:
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.FINISHED,
|
||||||
|
status_text="Nothing to clear",
|
||||||
|
result="System is clear",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.RUNNING,
|
||||||
|
progress=0,
|
||||||
|
status_text=f"Found {dead_packages} packages to remove!",
|
||||||
|
)
|
||||||
|
|
||||||
|
stream = run_nix_collect_garbage()
|
||||||
|
try:
|
||||||
|
process_stream(job, stream, dead_packages)
|
||||||
|
except ShellException as error:
|
||||||
|
Jobs.update(
|
||||||
|
job=job,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
status_text=COMPLETED_WITH_ERROR,
|
||||||
|
error=RESULT_WAS_NOT_FOUND_ERROR,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def start_nix_collect_garbage() -> Job:
|
||||||
|
job = Jobs.add(
|
||||||
|
type_id="maintenance.collect_nix_garbage",
|
||||||
|
name="Collect garbage",
|
||||||
|
description="Cleaning up unused packages",
|
||||||
|
)
|
||||||
|
|
||||||
|
calculate_and_clear_dead_paths(job=job)
|
||||||
|
|
||||||
|
return job
|
|
@ -1,5 +1,14 @@
|
||||||
|
from os import environ
|
||||||
|
|
||||||
from selfprivacy_api.utils.huey import huey
|
from selfprivacy_api.utils.huey import huey
|
||||||
from selfprivacy_api.jobs.test import test_job
|
|
||||||
from selfprivacy_api.backup.tasks import *
|
from selfprivacy_api.backup.tasks import *
|
||||||
from selfprivacy_api.services.tasks import move_service
|
from selfprivacy_api.services.tasks import move_service
|
||||||
from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
|
from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
|
||||||
|
|
||||||
|
from selfprivacy_api.jobs.test import test_job
|
||||||
|
from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths
|
||||||
|
|
||||||
|
|
||||||
|
if environ.get("TEST_MODE"):
|
||||||
|
from tests.test_huey import sum
|
||||||
|
|
|
@ -1,16 +1,24 @@
|
||||||
"""MiniHuey singleton."""
|
"""MiniHuey singleton."""
|
||||||
import os
|
from os import environ
|
||||||
from huey import SqliteHuey
|
from huey import RedisHuey
|
||||||
|
|
||||||
|
from selfprivacy_api.utils.redis_pool import RedisPool
|
||||||
|
|
||||||
|
HUEY_DATABASE_NUMBER = 10
|
||||||
|
|
||||||
|
|
||||||
|
def immediate() -> bool:
|
||||||
|
if environ.get("HUEY_QUEUES_FOR_TESTS"):
|
||||||
|
return False
|
||||||
|
if environ.get("TEST_MODE"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
HUEY_DATABASE = "/etc/selfprivacy/tasks.db"
|
|
||||||
|
|
||||||
# Singleton instance containing the huey database.
|
# Singleton instance containing the huey database.
|
||||||
|
huey = RedisHuey(
|
||||||
test_mode = os.environ.get("TEST_MODE")
|
|
||||||
|
|
||||||
huey = SqliteHuey(
|
|
||||||
"selfprivacy-api",
|
"selfprivacy-api",
|
||||||
filename=HUEY_DATABASE if not test_mode else None,
|
url=RedisPool.connection_url(dbnumber=HUEY_DATABASE_NUMBER),
|
||||||
immediate=test_mode == "true",
|
immediate=immediate(),
|
||||||
utc=True,
|
utc=True,
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
"""
|
"""
|
||||||
Redis pool module for selfprivacy_api
|
Redis pool module for selfprivacy_api
|
||||||
"""
|
"""
|
||||||
from os import environ
|
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
|
from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
|
||||||
|
|
||||||
REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
|
REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
|
||||||
|
@ -14,20 +14,20 @@ class RedisPool(metaclass=SingletonMetaclass):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
if "USE_REDIS_PORT" in environ:
|
self._pool = redis.ConnectionPool.from_url(
|
||||||
self._pool = redis.ConnectionPool(
|
RedisPool.connection_url(dbnumber=0),
|
||||||
host="127.0.0.1",
|
decode_responses=True,
|
||||||
port=int(environ["USE_REDIS_PORT"]),
|
)
|
||||||
decode_responses=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
else:
|
|
||||||
self._pool = redis.ConnectionPool.from_url(
|
|
||||||
f"unix://{REDIS_SOCKET}",
|
|
||||||
decode_responses=True,
|
|
||||||
)
|
|
||||||
self._pubsub_connection = self.get_connection()
|
self._pubsub_connection = self.get_connection()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def connection_url(dbnumber: int) -> str:
|
||||||
|
"""
|
||||||
|
redis://[[username]:[password]]@localhost:6379/0
|
||||||
|
unix://[username@]/path/to/socket.sock?db=0[&password=password]
|
||||||
|
"""
|
||||||
|
return f"unix://{REDIS_SOCKET}?db={dbnumber}"
|
||||||
|
|
||||||
def get_connection(self):
|
def get_connection(self):
|
||||||
"""
|
"""
|
||||||
Get a connection from the pool.
|
Get a connection from the pool.
|
||||||
|
|
|
@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="selfprivacy_api",
|
name="selfprivacy_api",
|
||||||
version="3.1.0",
|
version="3.2.1",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
scripts=[
|
scripts=[
|
||||||
"selfprivacy_api/app.py",
|
"selfprivacy_api/app.py",
|
||||||
|
|
|
@ -99,23 +99,14 @@ def generic_userdata(mocker, tmpdir):
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def huey_database(mocker, shared_datadir):
|
def client(redis_repo_with_tokens):
|
||||||
"""Mock huey database."""
|
|
||||||
mock = mocker.patch(
|
|
||||||
"selfprivacy_api.utils.huey.HUEY_DATABASE", shared_datadir / "huey.db"
|
|
||||||
)
|
|
||||||
return mock
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def client(huey_database, redis_repo_with_tokens):
|
|
||||||
from selfprivacy_api.app import app
|
from selfprivacy_api.app import app
|
||||||
|
|
||||||
return TestClient(app)
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def authorized_client(huey_database, redis_repo_with_tokens):
|
def authorized_client(redis_repo_with_tokens):
|
||||||
"""Authorized test client fixture."""
|
"""Authorized test client fixture."""
|
||||||
from selfprivacy_api.app import app
|
from selfprivacy_api.app import app
|
||||||
|
|
||||||
|
@ -127,7 +118,7 @@ def authorized_client(huey_database, redis_repo_with_tokens):
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def wrong_auth_client(huey_database, redis_repo_with_tokens):
|
def wrong_auth_client(redis_repo_with_tokens):
|
||||||
"""Wrong token test client fixture."""
|
"""Wrong token test client fixture."""
|
||||||
from selfprivacy_api.app import app
|
from selfprivacy_api.app import app
|
||||||
|
|
||||||
|
@ -217,5 +208,7 @@ def dummy_service(
|
||||||
service.enable()
|
service.enable()
|
||||||
yield service
|
yield service
|
||||||
|
|
||||||
# cleanup because apparently it matters wrt tasks
|
# Cleanup because apparently it matters wrt tasks
|
||||||
services.services.remove(service)
|
# Some tests may remove it from the list intentionally, this is fine
|
||||||
|
if service in services.services:
|
||||||
|
services.services.remove(service)
|
||||||
|
|
|
@ -3,6 +3,8 @@ from tests.test_backup import backups
|
||||||
from tests.common import generate_backup_query
|
from tests.common import generate_backup_query
|
||||||
|
|
||||||
|
|
||||||
|
import selfprivacy_api.services as all_services
|
||||||
|
from selfprivacy_api.services import get_service_by_id
|
||||||
from selfprivacy_api.graphql.common_types.service import service_to_graphql_service
|
from selfprivacy_api.graphql.common_types.service import service_to_graphql_service
|
||||||
from selfprivacy_api.graphql.common_types.backup import (
|
from selfprivacy_api.graphql.common_types.backup import (
|
||||||
_AutobackupQuotas,
|
_AutobackupQuotas,
|
||||||
|
@ -143,6 +145,7 @@ allSnapshots {
|
||||||
id
|
id
|
||||||
service {
|
service {
|
||||||
id
|
id
|
||||||
|
displayName
|
||||||
}
|
}
|
||||||
createdAt
|
createdAt
|
||||||
reason
|
reason
|
||||||
|
@ -306,6 +309,20 @@ def test_snapshots_empty(authorized_client, dummy_service, backups):
|
||||||
assert snaps == []
|
assert snaps == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshots_orphaned_service(authorized_client, dummy_service, backups):
|
||||||
|
api_backup(authorized_client, dummy_service)
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 1
|
||||||
|
|
||||||
|
all_services.services.remove(dummy_service)
|
||||||
|
assert get_service_by_id(dummy_service.get_id()) is None
|
||||||
|
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 1
|
||||||
|
assert "Orphaned" in snaps[0]["service"]["displayName"]
|
||||||
|
assert dummy_service.get_id() in snaps[0]["service"]["displayName"]
|
||||||
|
|
||||||
|
|
||||||
def test_start_backup(authorized_client, dummy_service, backups):
|
def test_start_backup(authorized_client, dummy_service, backups):
|
||||||
response = api_backup(authorized_client, dummy_service)
|
response = api_backup(authorized_client, dummy_service)
|
||||||
data = get_data(response)["backup"]["startBackup"]
|
data = get_data(response)["backup"]["startBackup"]
|
||||||
|
|
|
@ -0,0 +1,229 @@
|
||||||
|
# pylint: disable=redefined-outer-name
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
# pylint: disable=missing-function-docstring
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from selfprivacy_api.utils.huey import huey
|
||||||
|
|
||||||
|
from selfprivacy_api.jobs import JobStatus, Jobs
|
||||||
|
from tests.test_graphql.common import (
|
||||||
|
get_data,
|
||||||
|
assert_ok,
|
||||||
|
assert_empty,
|
||||||
|
)
|
||||||
|
|
||||||
|
from selfprivacy_api.jobs.nix_collect_garbage import (
|
||||||
|
get_dead_packages,
|
||||||
|
parse_line,
|
||||||
|
ShellException,
|
||||||
|
)
|
||||||
|
|
||||||
|
OUTPUT_PRINT_DEAD = """
|
||||||
|
finding garbage collector roots...
|
||||||
|
determining live/dead paths...
|
||||||
|
/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7
|
||||||
|
/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2
|
||||||
|
/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21
|
||||||
|
/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv
|
||||||
|
/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
OUTPUT_COLLECT_GARBAGE = """
|
||||||
|
removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
|
||||||
|
finding garbage collector roots...
|
||||||
|
deleting garbage...
|
||||||
|
deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7'
|
||||||
|
deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2'
|
||||||
|
deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21'
|
||||||
|
deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv'
|
||||||
|
deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook'
|
||||||
|
deleting unused links...
|
||||||
|
note: currently hard linking saves -0.00 MiB
|
||||||
|
190 store paths deleted, 425.51 MiB freed
|
||||||
|
"""
|
||||||
|
|
||||||
|
OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """
|
||||||
|
removing old generations of profile /nix/var/nix/profiles/per-user/def/profile
|
||||||
|
removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
|
||||||
|
finding garbage collector roots...
|
||||||
|
deleting garbage...
|
||||||
|
deleting unused links...
|
||||||
|
note: currently hard linking saves 0.00 MiB
|
||||||
|
0 store paths deleted, 0.00 MiB freed
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_line():
|
||||||
|
txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed"
|
||||||
|
|
||||||
|
job = Jobs.add(
|
||||||
|
name="name",
|
||||||
|
type_id="parse_line",
|
||||||
|
description="description",
|
||||||
|
)
|
||||||
|
|
||||||
|
output = parse_line(job, txt)
|
||||||
|
assert output.result == "425.51 MiB have been cleared"
|
||||||
|
assert output.status == JobStatus.FINISHED
|
||||||
|
assert output.error is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_line_with_blank_line():
|
||||||
|
txt = ""
|
||||||
|
job = Jobs.add(
|
||||||
|
name="name",
|
||||||
|
type_id="parse_line",
|
||||||
|
description="description",
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ShellException):
|
||||||
|
output = parse_line(job, txt)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_dead_packages():
|
||||||
|
assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_dead_packages_zero():
|
||||||
|
assert get_dead_packages("") == (0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
RUN_NIX_COLLECT_GARBAGE_MUTATION = """
|
||||||
|
mutation CollectGarbage {
|
||||||
|
system {
|
||||||
|
nixCollectGarbage {
|
||||||
|
success
|
||||||
|
message
|
||||||
|
code
|
||||||
|
job {
|
||||||
|
uid,
|
||||||
|
typeId,
|
||||||
|
name,
|
||||||
|
description,
|
||||||
|
status,
|
||||||
|
statusText,
|
||||||
|
progress,
|
||||||
|
createdAt,
|
||||||
|
updatedAt,
|
||||||
|
finishedAt,
|
||||||
|
error,
|
||||||
|
result,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_graphql_nix_collect_garbage(authorized_client, fp):
|
||||||
|
assert huey.immediate is True
|
||||||
|
|
||||||
|
fp.register(
|
||||||
|
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
|
||||||
|
stdout="",
|
||||||
|
)
|
||||||
|
fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
|
||||||
|
fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
|
||||||
|
|
||||||
|
response = authorized_client.post(
|
||||||
|
"/graphql",
|
||||||
|
json={
|
||||||
|
"query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
output = get_data(response)["system"]["nixCollectGarbage"]
|
||||||
|
assert_ok(output)
|
||||||
|
assert output["job"] is not None
|
||||||
|
assert output["job"]["status"] == "FINISHED"
|
||||||
|
assert output["job"]["error"] is None
|
||||||
|
|
||||||
|
assert (
|
||||||
|
fp.call_count(
|
||||||
|
[
|
||||||
|
"nix-env",
|
||||||
|
"-p",
|
||||||
|
"/nix/var/nix/profiles/system",
|
||||||
|
"--delete-generations old",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
|
||||||
|
assert fp.call_count(["nix-store", "--gc"]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_graphql_nix_collect_garbage_return_zero_trash(authorized_client, fp):
|
||||||
|
assert huey.immediate is True
|
||||||
|
|
||||||
|
fp.register(
|
||||||
|
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
|
||||||
|
stdout="",
|
||||||
|
)
|
||||||
|
fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
|
||||||
|
fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH)
|
||||||
|
|
||||||
|
response = authorized_client.post(
|
||||||
|
"/graphql",
|
||||||
|
json={
|
||||||
|
"query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
output = get_data(response)["system"]["nixCollectGarbage"]
|
||||||
|
assert_ok(output)
|
||||||
|
assert output["job"] is not None
|
||||||
|
assert output["job"]["status"] == "FINISHED"
|
||||||
|
assert output["job"]["error"] is None
|
||||||
|
|
||||||
|
assert (
|
||||||
|
fp.call_count(
|
||||||
|
[
|
||||||
|
"nix-env",
|
||||||
|
"-p",
|
||||||
|
"/nix/var/nix/profiles/system",
|
||||||
|
"--delete-generations old",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
|
||||||
|
assert fp.call_count(["nix-store", "--gc"]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_graphql_nix_collect_garbage_not_authorized_client(client, fp):
|
||||||
|
assert huey.immediate is True
|
||||||
|
|
||||||
|
fp.register(
|
||||||
|
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
|
||||||
|
stdout="",
|
||||||
|
)
|
||||||
|
fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
|
||||||
|
fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
"/graphql",
|
||||||
|
json={
|
||||||
|
"query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert_empty(response)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
fp.call_count(
|
||||||
|
[
|
||||||
|
"nix-env",
|
||||||
|
"-p",
|
||||||
|
"/nix/var/nix/profiles/system",
|
||||||
|
"--delete-generations old",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
== 0
|
||||||
|
)
|
||||||
|
assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 0
|
||||||
|
assert fp.call_count(["nix-store", "--gc"]) == 0
|
|
@ -0,0 +1,132 @@
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from subprocess import Popen, check_output, TimeoutExpired
|
||||||
|
from os import environ, path, set_blocking
|
||||||
|
from io import BufferedReader
|
||||||
|
from huey.exceptions import HueyException
|
||||||
|
|
||||||
|
from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
|
||||||
|
from selfprivacy_api.utils.redis_pool import RedisPool, REDIS_SOCKET
|
||||||
|
|
||||||
|
|
||||||
|
@huey.task()
|
||||||
|
def sum(a: int, b: int) -> int:
|
||||||
|
return a + b
|
||||||
|
|
||||||
|
|
||||||
|
def reset_huey_storage():
|
||||||
|
huey.storage = huey.create_storage()
|
||||||
|
|
||||||
|
|
||||||
|
def flush_huey_redis_forcefully():
|
||||||
|
url = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
|
||||||
|
|
||||||
|
pool = redis.ConnectionPool.from_url(url, decode_responses=True)
|
||||||
|
connection = redis.Redis(connection_pool=pool)
|
||||||
|
connection.flushdb()
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: may be useful in other places too, move to utils/ tests common if using it somewhere
|
||||||
|
def read_all_ready_output(stream: BufferedReader) -> str:
|
||||||
|
set_blocking(stream.fileno(), False)
|
||||||
|
output: List[bytes] = []
|
||||||
|
while True:
|
||||||
|
line = stream.readline()
|
||||||
|
raise ValueError(line)
|
||||||
|
if line == b"":
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
output.append(line)
|
||||||
|
|
||||||
|
set_blocking(stream.fileno(), True)
|
||||||
|
|
||||||
|
result = b"".join(output)
|
||||||
|
return result.decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def not_immediate():
|
||||||
|
assert environ["TEST_MODE"] == "true"
|
||||||
|
|
||||||
|
old_immediate = huey.immediate
|
||||||
|
environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
|
||||||
|
huey.immediate = False
|
||||||
|
assert huey.immediate is False
|
||||||
|
|
||||||
|
yield
|
||||||
|
|
||||||
|
del environ["HUEY_QUEUES_FOR_TESTS"]
|
||||||
|
huey.immediate = old_immediate
|
||||||
|
assert huey.immediate == old_immediate
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def huey_socket_consumer(not_immediate):
|
||||||
|
"""
|
||||||
|
Same as above, but with socketed redis
|
||||||
|
"""
|
||||||
|
|
||||||
|
flush_huey_redis_forcefully()
|
||||||
|
command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
|
||||||
|
|
||||||
|
# First assert that consumer does not fail by itself
|
||||||
|
# Idk yet how to do it more elegantly
|
||||||
|
try:
|
||||||
|
check_output(command, timeout=2)
|
||||||
|
except TimeoutExpired:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Then open it for real
|
||||||
|
consumer_handle = Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
|
||||||
|
assert path.exists(REDIS_SOCKET)
|
||||||
|
|
||||||
|
yield consumer_handle
|
||||||
|
|
||||||
|
consumer_handle.kill()
|
||||||
|
|
||||||
|
|
||||||
|
def test_huey_over_redis_socket(huey_socket_consumer):
|
||||||
|
assert huey.immediate is False
|
||||||
|
assert immediate() is False
|
||||||
|
|
||||||
|
assert "unix" in RedisPool.connection_url(HUEY_DATABASE_NUMBER)
|
||||||
|
try:
|
||||||
|
assert (
|
||||||
|
RedisPool.connection_url(HUEY_DATABASE_NUMBER)
|
||||||
|
in huey.storage_kwargs.values()
|
||||||
|
)
|
||||||
|
except AssertionError:
|
||||||
|
raise ValueError(
|
||||||
|
"our test-side huey does not connect over socket: ", huey.storage_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
result = sum(2, 5)
|
||||||
|
try:
|
||||||
|
assert result(blocking=True, timeout=100) == 7
|
||||||
|
|
||||||
|
except HueyException as error:
|
||||||
|
if "timed out" in str(error):
|
||||||
|
output = read_all_ready_output(huey_socket_consumer.stdout)
|
||||||
|
errorstream = read_all_ready_output(huey_socket_consumer.stderr)
|
||||||
|
raise TimeoutError(
|
||||||
|
f"Huey timed out: {str(error)}",
|
||||||
|
f"Consumer output: {output}",
|
||||||
|
f"Consumer errorstream: {errorstream}",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise error
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.xfail(reason="cannot yet schedule with sockets for some reason")
|
||||||
|
def test_huey_schedule(huey_queues_socket):
|
||||||
|
# We do not schedule tasks anywhere, but concerning that it fails.
|
||||||
|
sum.schedule((2, 5), delay=10)
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert len(huey.scheduled()) == 1
|
||||||
|
except AssertionError:
|
||||||
|
raise ValueError("have wrong amount of scheduled tasks", huey.scheduled())
|
Loading…
Reference in New Issue