feat: Add "OTHER" as a server provider

We should allow manual SelfPrivacy installations on unsupported server providers. The ServerProvider enum is one of the gatekeepers that prevent this and we can change it easily as not much server-side logic rely on this.

The next step would be manual DNS management, but it would be much more involved than just adding the enum value.

selfprivacy_api/dependencies.py

@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "3.1.0"
+    return "3.2.1"

selfprivacy_api/graphql/mutations/system_mutations.py

@@ -8,9 +8,12 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
     GenericJobMutationReturn,
     GenericMutationReturn,
     MutationReturnInterface,
+    GenericJobMutationReturn,
 )
 
 import selfprivacy_api.actions.system as system_actions
+from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
+from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
 import selfprivacy_api.actions.ssh as ssh_actions
 
 
@@ -195,3 +198,14 @@ class SystemMutations:
             message=f"Failed to pull repository changes:\n{result.data}",
             code=500,
         )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def nix_collect_garbage(self) -> GenericJobMutationReturn:
+        job = start_nix_collect_garbage()
+
+        return GenericJobMutationReturn(
+            success=True,
+            code=200,
+            message="Garbage collector started...",
+            job=job_to_api_job(job),
+        )

selfprivacy_api/graphql/queries/backup.py

@@ -34,6 +34,24 @@ class BackupConfiguration:
     location_id: typing.Optional[str]
 
 
+# TODO: Ideally this should not be done in API but making an internal Service requires more work
+# than to make an API record about a service
+def tombstone_service(service_id: str) -> Service:
+    return Service(
+        id=service_id,
+        display_name=f"{service_id} (Orphaned)",
+        description="",
+        svg_icon="",
+        is_movable=False,
+        is_required=False,
+        is_enabled=False,
+        status=ServiceStatusEnum.OFF,
+        url=None,
+        can_be_backed_up=False,
+        backup_description="",
+    )
+
+
 @strawberry.type
 class Backup:
     @strawberry.field
@@ -55,27 +73,21 @@ class Backup:
         result = []
         snapshots = Backups.get_all_snapshots()
         for snap in snapshots:
+            api_service = None
             service = get_service_by_id(snap.service_name)
+
             if service is None:
-                service = Service(
+                api_service = tombstone_service(snap.service_name)
-                    id=snap.service_name,
-                    display_name=f"{snap.service_name} (Orphaned)",
-                    description="",
-                    svg_icon="",
-                    is_movable=False,
-                    is_required=False,
-                    is_enabled=False,
-                    status=ServiceStatusEnum.OFF,
-                    url=None,
-                    dns_records=None,
-                    can_be_backed_up=False,
-                    backup_description="",
-                )
             else:
-                service = service_to_graphql_service(service)
+                api_service = service_to_graphql_service(service)
+            if api_service is None:
+                raise NotImplementedError(
+                    f"Could not construct API Service record for:{snap.service_name}. This should be unreachable and is a bug if you see it."
+                )
+
             graphql_snap = SnapshotInfo(
                 id=snap.id,
-                service=service,
+                service=api_service,
                 created_at=snap.created_at,
                 reason=snap.reason,
             )

selfprivacy_api/graphql/queries/providers.py

@@ -14,6 +14,7 @@ class DnsProvider(Enum):
 class ServerProvider(Enum):
     HETZNER = "HETZNER"
     DIGITALOCEAN = "DIGITALOCEAN"
+    OTHER = "OTHER"
 
 
 @strawberry.enum

selfprivacy_api/jobs/migrate_to_binds.py

@@ -67,8 +67,8 @@ def move_folder(
 
     try:
         data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
-    except Exception as e:
+    except Exception as error:
-        print(f"Error creating data path: {e}")
+        print(f"Error creating data path: {error}")
         return
 
     try:

selfprivacy_api/jobs/nix_collect_garbage.py

@@ -0,0 +1,147 @@
+import re
+import subprocess
+from typing import Tuple, Iterable
+
+from selfprivacy_api.utils.huey import huey
+
+from selfprivacy_api.jobs import JobStatus, Jobs, Job
+
+
+class ShellException(Exception):
+    """Shell-related errors"""
+
+
+COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
+RESULT_WAS_NOT_FOUND_ERROR = (
+    "We are sorry, garbage collection result was not found. "
+    "Something went wrong, please report this to the support chat."
+)
+CLEAR_COMPLETED = "Garbage collection completed."
+
+
+def delete_old_gens_and_return_dead_report() -> str:
+    subprocess.run(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        check=False,
+    )
+
+    result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
+        "utf-8"
+    )
+
+    return " " if result is None else result
+
+
+def run_nix_collect_garbage() -> Iterable[bytes]:
+    process = subprocess.Popen(
+        ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    return process.stdout if process.stdout else iter([])
+
+
+def parse_line(job: Job, line: str) -> Job:
+    """
+    We parse the string for the presence of a final line,
+    with the final amount of space cleared.
+    Simply put, we're just looking for a similar string:
+    "1537 store paths deleted, 339.84 MiB freed".
+    """
+    pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
+    match = re.search(pattern, line)
+
+    if match is None:
+        raise ShellException("nix returned gibberish output")
+
+    else:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text=CLEAR_COMPLETED,
+            result=f"{match.group(0)} have been cleared",
+        )
+    return job
+
+
+def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
+    completed_packages = 0
+    prev_progress = 0
+
+    for line in stream:
+        line = line.decode("utf-8")
+
+        if "deleting '/nix/store/" in line:
+            completed_packages += 1
+            percent = int((completed_packages / total_dead_packages) * 100)
+
+            if percent - prev_progress >= 5:
+                Jobs.update(
+                    job=job,
+                    status=JobStatus.RUNNING,
+                    progress=percent,
+                    status_text="Cleaning...",
+                )
+                prev_progress = percent
+
+        elif "store paths deleted," in line:
+            parse_line(job, line)
+
+
+def get_dead_packages(output) -> Tuple[int, float]:
+    dead = len(re.findall("/nix/store/", output))
+    percent = 0
+    if dead != 0:
+        percent = 100 / dead
+    return dead, percent
+
+
+@huey.task()
+def calculate_and_clear_dead_paths(job: Job):
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text="Calculate the number of dead packages...",
+    )
+
+    dead_packages, package_equal_to_percent = get_dead_packages(
+        delete_old_gens_and_return_dead_report()
+    )
+
+    if dead_packages == 0:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text="Nothing to clear",
+            result="System is clear",
+        )
+        return True
+
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text=f"Found {dead_packages} packages to remove!",
+    )
+
+    stream = run_nix_collect_garbage()
+    try:
+        process_stream(job, stream, dead_packages)
+    except ShellException as error:
+        Jobs.update(
+            job=job,
+            status=JobStatus.ERROR,
+            status_text=COMPLETED_WITH_ERROR,
+            error=RESULT_WAS_NOT_FOUND_ERROR,
+        )
+
+
+def start_nix_collect_garbage() -> Job:
+    job = Jobs.add(
+        type_id="maintenance.collect_nix_garbage",
+        name="Collect garbage",
+        description="Cleaning up unused packages",
+    )
+
+    calculate_and_clear_dead_paths(job=job)
+
+    return job

selfprivacy_api/task_registry.py

@@ -7,6 +7,8 @@ from selfprivacy_api.services.tasks import move_service
 from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
 
 from selfprivacy_api.jobs.test import test_job
+from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths
+
 
 if environ.get("TEST_MODE"):
     from tests.test_huey import sum

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="selfprivacy_api",
-    version="3.1.0",
+    version="3.2.1",
     packages=find_packages(),
     scripts=[
         "selfprivacy_api/app.py",

tests/conftest.py

@@ -208,5 +208,7 @@ def dummy_service(
     service.enable()
     yield service
 
-    # cleanup because apparently it matters wrt tasks
+    # Cleanup because apparently it matters wrt tasks
-    services.services.remove(service)
+    # Some tests may remove it from the list intentionally, this is fine
+    if service in services.services:
+        services.services.remove(service)

tests/test_graphql/test_api_backup.py

@@ -3,6 +3,8 @@ from tests.test_backup import backups
 from tests.common import generate_backup_query
 
 
+import selfprivacy_api.services as all_services
+from selfprivacy_api.services import get_service_by_id
 from selfprivacy_api.graphql.common_types.service import service_to_graphql_service
 from selfprivacy_api.graphql.common_types.backup import (
     _AutobackupQuotas,
@@ -143,6 +145,7 @@ allSnapshots {
     id
     service {
         id
+        displayName
     }
     createdAt
     reason
@@ -306,6 +309,20 @@ def test_snapshots_empty(authorized_client, dummy_service, backups):
     assert snaps == []
 
 
+def test_snapshots_orphaned_service(authorized_client, dummy_service, backups):
+    api_backup(authorized_client, dummy_service)
+    snaps = api_snapshots(authorized_client)
+    assert len(snaps) == 1
+
+    all_services.services.remove(dummy_service)
+    assert get_service_by_id(dummy_service.get_id()) is None
+
+    snaps = api_snapshots(authorized_client)
+    assert len(snaps) == 1
+    assert "Orphaned" in snaps[0]["service"]["displayName"]
+    assert dummy_service.get_id() in snaps[0]["service"]["displayName"]
+
+
 def test_start_backup(authorized_client, dummy_service, backups):
     response = api_backup(authorized_client, dummy_service)
     data = get_data(response)["backup"]["startBackup"]

tests/test_graphql/test_nix_collect_garbage.py

@@ -0,0 +1,229 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=missing-function-docstring
+
+import pytest
+from selfprivacy_api.utils.huey import huey
+
+from selfprivacy_api.jobs import JobStatus, Jobs
+from tests.test_graphql.common import (
+    get_data,
+    assert_ok,
+    assert_empty,
+)
+
+from selfprivacy_api.jobs.nix_collect_garbage import (
+    get_dead_packages,
+    parse_line,
+    ShellException,
+)
+
+OUTPUT_PRINT_DEAD = """
+finding garbage collector roots...
+determining live/dead paths...
+/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7
+/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2
+/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21
+/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv
+/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook
+"""
+
+
+OUTPUT_COLLECT_GARBAGE = """
+removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
+finding garbage collector roots...
+deleting garbage...
+deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7'
+deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2'
+deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21'
+deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv'
+deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook'
+deleting unused links...
+note: currently hard linking saves -0.00 MiB
+190 store paths deleted, 425.51 MiB freed
+"""
+
+OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """
+removing old generations of profile /nix/var/nix/profiles/per-user/def/profile
+removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
+finding garbage collector roots...
+deleting garbage...
+deleting unused links...
+note: currently hard linking saves 0.00 MiB
+0 store paths deleted, 0.00 MiB freed
+"""
+
+
+# ---
+
+
+def test_parse_line():
+    txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed"
+
+    job = Jobs.add(
+        name="name",
+        type_id="parse_line",
+        description="description",
+    )
+
+    output = parse_line(job, txt)
+    assert output.result == "425.51 MiB have been cleared"
+    assert output.status == JobStatus.FINISHED
+    assert output.error is None
+
+
+def test_parse_line_with_blank_line():
+    txt = ""
+    job = Jobs.add(
+        name="name",
+        type_id="parse_line",
+        description="description",
+    )
+
+    with pytest.raises(ShellException):
+        output = parse_line(job, txt)
+
+
+def test_get_dead_packages():
+    assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0)
+
+
+def test_get_dead_packages_zero():
+    assert get_dead_packages("") == (0, 0)
+
+
+RUN_NIX_COLLECT_GARBAGE_MUTATION = """
+mutation CollectGarbage {
+    system {
+        nixCollectGarbage {
+            success
+            message
+            code
+            job {
+                uid,
+                typeId,
+                name,
+                description,
+                status,
+                statusText,
+                progress,
+                createdAt,
+                updatedAt,
+                finishedAt,
+                error,
+                result,
+            }
+        }
+    }
+}
+"""
+
+
+def test_graphql_nix_collect_garbage(authorized_client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    output = get_data(response)["system"]["nixCollectGarbage"]
+    assert_ok(output)
+    assert output["job"] is not None
+    assert output["job"]["status"] == "FINISHED"
+    assert output["job"]["error"] is None
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 1
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
+    assert fp.call_count(["nix-store", "--gc"]) == 1
+
+
+def test_graphql_nix_collect_garbage_return_zero_trash(authorized_client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH)
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    output = get_data(response)["system"]["nixCollectGarbage"]
+    assert_ok(output)
+    assert output["job"] is not None
+    assert output["job"]["status"] == "FINISHED"
+    assert output["job"]["error"] is None
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 1
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
+    assert fp.call_count(["nix-store", "--gc"]) == 1
+
+
+def test_graphql_nix_collect_garbage_not_authorized_client(client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
+
+    response = client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    assert_empty(response)
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 0
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 0
+    assert fp.call_count(["nix-store", "--gc"]) == 0