chore: Bump version to 3.2.1

Continuation of the broken #21

Co-authored-by: dettlaff <dettlaff@riseup.net>
Co-authored-by: def <dettlaff@riseup.net>
Co-authored-by: Houkime <>
Reviewed-on: #112
Reviewed-by: houkime <houkime@protonmail.com>

selfprivacy_api/dependencies.py

@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "3.1.0"
+    return "3.2.1"

selfprivacy_api/graphql/mutations/system_mutations.py

@@ -8,9 +8,12 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
     GenericJobMutationReturn,
     GenericMutationReturn,
     MutationReturnInterface,
+    GenericJobMutationReturn,
 )
 
 import selfprivacy_api.actions.system as system_actions
+from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
+from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
 import selfprivacy_api.actions.ssh as ssh_actions
 
 
@@ -195,3 +198,14 @@ class SystemMutations:
             message=f"Failed to pull repository changes:\n{result.data}",
             code=500,
         )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def nix_collect_garbage(self) -> GenericJobMutationReturn:
+        job = start_nix_collect_garbage()
+
+        return GenericJobMutationReturn(
+            success=True,
+            code=200,
+            message="Garbage collector started...",
+            job=job_to_api_job(job),
+        )

selfprivacy_api/graphql/queries/backup.py

@@ -34,6 +34,24 @@ class BackupConfiguration:
     location_id: typing.Optional[str]
 
 
+# TODO: Ideally this should not be done in API but making an internal Service requires more work
+# than to make an API record about a service
+def tombstone_service(service_id: str) -> Service:
+    return Service(
+        id=service_id,
+        display_name=f"{service_id} (Orphaned)",
+        description="",
+        svg_icon="",
+        is_movable=False,
+        is_required=False,
+        is_enabled=False,
+        status=ServiceStatusEnum.OFF,
+        url=None,
+        can_be_backed_up=False,
+        backup_description="",
+    )
+
+
 @strawberry.type
 class Backup:
     @strawberry.field
@@ -55,27 +73,21 @@ class Backup:
         result = []
         snapshots = Backups.get_all_snapshots()
         for snap in snapshots:
+            api_service = None
             service = get_service_by_id(snap.service_name)
+
             if service is None:
-                service = Service(
-                    id=snap.service_name,
-                    display_name=f"{snap.service_name} (Orphaned)",
-                    description="",
-                    svg_icon="",
-                    is_movable=False,
-                    is_required=False,
-                    is_enabled=False,
-                    status=ServiceStatusEnum.OFF,
-                    url=None,
-                    dns_records=None,
-                    can_be_backed_up=False,
-                    backup_description="",
-                )
+                api_service = tombstone_service(snap.service_name)
             else:
-                service = service_to_graphql_service(service)
+                api_service = service_to_graphql_service(service)
+            if api_service is None:
+                raise NotImplementedError(
+                    f"Could not construct API Service record for:{snap.service_name}. This should be unreachable and is a bug if you see it."
+                )
+
             graphql_snap = SnapshotInfo(
                 id=snap.id,
-                service=service,
+                service=api_service,
                 created_at=snap.created_at,
                 reason=snap.reason,
             )

selfprivacy_api/jobs/migrate_to_binds.py

@@ -67,8 +67,8 @@ def move_folder(
 
     try:
         data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
-    except Exception as e:
-        print(f"Error creating data path: {e}")
+    except Exception as error:
+        print(f"Error creating data path: {error}")
         return
 
     try:

selfprivacy_api/jobs/nix_collect_garbage.py

@@ -0,0 +1,147 @@
+import re
+import subprocess
+from typing import Tuple, Iterable
+
+from selfprivacy_api.utils.huey import huey
+
+from selfprivacy_api.jobs import JobStatus, Jobs, Job
+
+
+class ShellException(Exception):
+    """Shell-related errors"""
+
+
+COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
+RESULT_WAS_NOT_FOUND_ERROR = (
+    "We are sorry, garbage collection result was not found. "
+    "Something went wrong, please report this to the support chat."
+)
+CLEAR_COMPLETED = "Garbage collection completed."
+
+
+def delete_old_gens_and_return_dead_report() -> str:
+    subprocess.run(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        check=False,
+    )
+
+    result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
+        "utf-8"
+    )
+
+    return " " if result is None else result
+
+
+def run_nix_collect_garbage() -> Iterable[bytes]:
+    process = subprocess.Popen(
+        ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    return process.stdout if process.stdout else iter([])
+
+
+def parse_line(job: Job, line: str) -> Job:
+    """
+    We parse the string for the presence of a final line,
+    with the final amount of space cleared.
+    Simply put, we're just looking for a similar string:
+    "1537 store paths deleted, 339.84 MiB freed".
+    """
+    pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
+    match = re.search(pattern, line)
+
+    if match is None:
+        raise ShellException("nix returned gibberish output")
+
+    else:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text=CLEAR_COMPLETED,
+            result=f"{match.group(0)} have been cleared",
+        )
+    return job
+
+
+def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
+    completed_packages = 0
+    prev_progress = 0
+
+    for line in stream:
+        line = line.decode("utf-8")
+
+        if "deleting '/nix/store/" in line:
+            completed_packages += 1
+            percent = int((completed_packages / total_dead_packages) * 100)
+
+            if percent - prev_progress >= 5:
+                Jobs.update(
+                    job=job,
+                    status=JobStatus.RUNNING,
+                    progress=percent,
+                    status_text="Cleaning...",
+                )
+                prev_progress = percent
+
+        elif "store paths deleted," in line:
+            parse_line(job, line)
+
+
+def get_dead_packages(output) -> Tuple[int, float]:
+    dead = len(re.findall("/nix/store/", output))
+    percent = 0
+    if dead != 0:
+        percent = 100 / dead
+    return dead, percent
+
+
+@huey.task()
+def calculate_and_clear_dead_paths(job: Job):
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text="Calculate the number of dead packages...",
+    )
+
+    dead_packages, package_equal_to_percent = get_dead_packages(
+        delete_old_gens_and_return_dead_report()
+    )
+
+    if dead_packages == 0:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text="Nothing to clear",
+            result="System is clear",
+        )
+        return True
+
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text=f"Found {dead_packages} packages to remove!",
+    )
+
+    stream = run_nix_collect_garbage()
+    try:
+        process_stream(job, stream, dead_packages)
+    except ShellException as error:
+        Jobs.update(
+            job=job,
+            status=JobStatus.ERROR,
+            status_text=COMPLETED_WITH_ERROR,
+            error=RESULT_WAS_NOT_FOUND_ERROR,
+        )
+
+
+def start_nix_collect_garbage() -> Job:
+    job = Jobs.add(
+        type_id="maintenance.collect_nix_garbage",
+        name="Collect garbage",
+        description="Cleaning up unused packages",
+    )
+
+    calculate_and_clear_dead_paths(job=job)
+
+    return job

selfprivacy_api/task_registry.py

@@ -7,6 +7,8 @@ from selfprivacy_api.services.tasks import move_service
 from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
 
 from selfprivacy_api.jobs.test import test_job
+from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths
+
 
 if environ.get("TEST_MODE"):
     from tests.test_huey import sum

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="selfprivacy_api",
-    version="3.1.0",
+    version="3.2.1",
     packages=find_packages(),
     scripts=[
         "selfprivacy_api/app.py",

tests/conftest.py

@@ -208,5 +208,7 @@ def dummy_service(
     service.enable()
     yield service
 
-    # cleanup because apparently it matters wrt tasks
+    # Cleanup because apparently it matters wrt tasks
+    # Some tests may remove it from the list intentionally, this is fine
+    if service in services.services:
         services.services.remove(service)

tests/test_graphql/test_api_backup.py

@@ -3,6 +3,8 @@ from tests.test_backup import backups
 from tests.common import generate_backup_query
 
 
+import selfprivacy_api.services as all_services
+from selfprivacy_api.services import get_service_by_id
 from selfprivacy_api.graphql.common_types.service import service_to_graphql_service
 from selfprivacy_api.graphql.common_types.backup import (
     _AutobackupQuotas,
@@ -143,6 +145,7 @@ allSnapshots {
     id
     service {
         id
+        displayName
     }
     createdAt
     reason
@@ -306,6 +309,20 @@ def test_snapshots_empty(authorized_client, dummy_service, backups):
     assert snaps == []
 
 
+def test_snapshots_orphaned_service(authorized_client, dummy_service, backups):
+    api_backup(authorized_client, dummy_service)
+    snaps = api_snapshots(authorized_client)
+    assert len(snaps) == 1
+
+    all_services.services.remove(dummy_service)
+    assert get_service_by_id(dummy_service.get_id()) is None
+
+    snaps = api_snapshots(authorized_client)
+    assert len(snaps) == 1
+    assert "Orphaned" in snaps[0]["service"]["displayName"]
+    assert dummy_service.get_id() in snaps[0]["service"]["displayName"]
+
+
 def test_start_backup(authorized_client, dummy_service, backups):
     response = api_backup(authorized_client, dummy_service)
     data = get_data(response)["backup"]["startBackup"]

tests/test_graphql/test_nix_collect_garbage.py

@@ -0,0 +1,229 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=missing-function-docstring
+
+import pytest
+from selfprivacy_api.utils.huey import huey
+
+from selfprivacy_api.jobs import JobStatus, Jobs
+from tests.test_graphql.common import (
+    get_data,
+    assert_ok,
+    assert_empty,
+)
+
+from selfprivacy_api.jobs.nix_collect_garbage import (
+    get_dead_packages,
+    parse_line,
+    ShellException,
+)
+
+OUTPUT_PRINT_DEAD = """
+finding garbage collector roots...
+determining live/dead paths...
+/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7
+/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2
+/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21
+/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv
+/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook
+"""
+
+
+OUTPUT_COLLECT_GARBAGE = """
+removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
+finding garbage collector roots...
+deleting garbage...
+deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7'
+deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2'
+deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21'
+deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv'
+deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook'
+deleting unused links...
+note: currently hard linking saves -0.00 MiB
+190 store paths deleted, 425.51 MiB freed
+"""
+
+OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """
+removing old generations of profile /nix/var/nix/profiles/per-user/def/profile
+removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
+finding garbage collector roots...
+deleting garbage...
+deleting unused links...
+note: currently hard linking saves 0.00 MiB
+0 store paths deleted, 0.00 MiB freed
+"""
+
+
+# ---
+
+
+def test_parse_line():
+    txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed"
+
+    job = Jobs.add(
+        name="name",
+        type_id="parse_line",
+        description="description",
+    )
+
+    output = parse_line(job, txt)
+    assert output.result == "425.51 MiB have been cleared"
+    assert output.status == JobStatus.FINISHED
+    assert output.error is None
+
+
+def test_parse_line_with_blank_line():
+    txt = ""
+    job = Jobs.add(
+        name="name",
+        type_id="parse_line",
+        description="description",
+    )
+
+    with pytest.raises(ShellException):
+        output = parse_line(job, txt)
+
+
+def test_get_dead_packages():
+    assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0)
+
+
+def test_get_dead_packages_zero():
+    assert get_dead_packages("") == (0, 0)
+
+
+RUN_NIX_COLLECT_GARBAGE_MUTATION = """
+mutation CollectGarbage {
+    system {
+        nixCollectGarbage {
+            success
+            message
+            code
+            job {
+                uid,
+                typeId,
+                name,
+                description,
+                status,
+                statusText,
+                progress,
+                createdAt,
+                updatedAt,
+                finishedAt,
+                error,
+                result,
+            }
+        }
+    }
+}
+"""
+
+
+def test_graphql_nix_collect_garbage(authorized_client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    output = get_data(response)["system"]["nixCollectGarbage"]
+    assert_ok(output)
+    assert output["job"] is not None
+    assert output["job"]["status"] == "FINISHED"
+    assert output["job"]["error"] is None
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 1
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
+    assert fp.call_count(["nix-store", "--gc"]) == 1
+
+
+def test_graphql_nix_collect_garbage_return_zero_trash(authorized_client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH)
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    output = get_data(response)["system"]["nixCollectGarbage"]
+    assert_ok(output)
+    assert output["job"] is not None
+    assert output["job"]["status"] == "FINISHED"
+    assert output["job"]["error"] is None
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 1
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
+    assert fp.call_count(["nix-store", "--gc"]) == 1
+
+
+def test_graphql_nix_collect_garbage_not_authorized_client(client, fp):
+    assert huey.immediate is True
+
+    fp.register(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        stdout="",
+    )
+    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
+    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
+
+    response = client.post(
+        "/graphql",
+        json={
+            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
+        },
+    )
+
+    assert_empty(response)
+
+    assert (
+        fp.call_count(
+            [
+                "nix-env",
+                "-p",
+                "/nix/var/nix/profiles/system",
+                "--delete-generations old",
+            ]
+        )
+        == 0
+    )
+    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 0
+    assert fp.call_count(["nix-store", "--gc"]) == 0