15 changed files with 14 additions and 515 deletions
--- a/.gitignore
+++ b/.gitignore
--- a/.mypy.ini
+++ b/.mypy.ini
@ -1,2 +0,0 @@
-[mypy]
-plugins = pydantic.mypy
--- a/README.md
+++ b/README.md
@ -1,8 +1,6 @@
 # SelfPrivacy GraphQL API which allows app to control your server

-![CI status](https://ci.selfprivacy.org/api/badges/SelfPrivacy/selfprivacy-rest-api/status.svg)
-
-## Build
+## build

 ```console
 $ nix build
@ -10,7 +8,7 @@ $ nix build

 In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.

-## Develop
+## develop

 ```console
 $ nix develop
@ -23,10 +21,10 @@ Type "help", "copyright", "credits" or "license" for more information.
 If you don't have experimental flakes enabled, you can use the following command:

 ```console
-$ nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
+nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
 ```

-## Testing
+## testing

 Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
 ```console
@ -63,7 +61,7 @@ $ TMPDIR=".nixos-vm-tmp-dir" nix run .#checks.x86_64-linux.default.driverInterac

 Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.

-## Dependencies and Dependant Modules
+## dependencies and dependant modules

 This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.

@ -87,6 +85,6 @@ $ nix flake metadata git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nix

 Nix code for NixOS service module for API is located in NixOS configuration repository.

-## Troubleshooting
+## troubleshooting

 Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.
--- a/selfprivacy_api/backup/init.py
+++ b/selfprivacy_api/backup/init.py
@ -259,7 +259,7 @@ class Backups:
                Backups._prune_auto_snaps(service)
            service.post_restore()
        except Exception as error:
-            Jobs.update(job, status=JobStatus.ERROR, error=str(error))
+            Jobs.update(job, status=JobStatus.ERROR, status_text=str(error))
            raise error

        Jobs.update(job, status=JobStatus.FINISHED)
--- a/selfprivacy_api/backup/backuppers/restic_backupper.py
+++ b/selfprivacy_api/backup/backuppers/restic_backupper.py
@ -172,21 +172,6 @@ class ResticBackupper(AbstractBackupper):

        return messages

-    @staticmethod
-    def _replace_in_array(array: List[str], target, replacement) -> None:
-        if target == "":
-            return
-
-        for i, value in enumerate(array):
-            if target in value:
-                array[i] = array[i].replace(target, replacement)
-
-    def _censor_command(self, command: List[str]) -> List[str]:
-        result = command.copy()
-        ResticBackupper._replace_in_array(result, self.key, "CENSORED")
-        ResticBackupper._replace_in_array(result, LocalBackupSecret.get(), "CENSORED")
-        return result
-
    @staticmethod
    def _get_backup_job(service_name: str) -> Optional[Job]:
        service = get_service_by_id(service_name)
@ -233,7 +218,7 @@ class ResticBackupper(AbstractBackupper):
                "Could not create a snapshot: ",
                str(error),
                "command: ",
-                self._censor_command(backup_command),
+                backup_command,
            ) from error

    @staticmethod
--- a/selfprivacy_api/backup/providers/init.py
+++ b/selfprivacy_api/backup/providers/init.py
@ -21,8 +21,6 @@ PROVIDER_MAPPING: dict[BackupProviderEnum, Type[AbstractBackupProvider]] = {
 def get_provider(
    provider_type: BackupProviderEnum,
 ) -> Type[AbstractBackupProvider]:
-    if provider_type not in PROVIDER_MAPPING.keys():
-        raise LookupError("could not look up provider", provider_type)
    return PROVIDER_MAPPING[provider_type]


--- a/selfprivacy_api/dependencies.py
+++ b/selfprivacy_api/dependencies.py
@ -27,4 +27,4 @@ async def get_token_header(

 def get_api_version() -> str:
    """Get API version"""
-    return "3.2.0"
+    return "3.1.0"
--- a/selfprivacy_api/graphql/mutations/system_mutations.py
+++ b/selfprivacy_api/graphql/mutations/system_mutations.py
@ -8,12 +8,9 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
    GenericJobMutationReturn,
    GenericMutationReturn,
    MutationReturnInterface,
-    GenericJobMutationReturn,
 )

 import selfprivacy_api.actions.system as system_actions
-from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
-from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
 import selfprivacy_api.actions.ssh as ssh_actions


@ -198,14 +195,3 @@ class SystemMutations:
            message=f"Failed to pull repository changes:\n{result.data}",
            code=500,
        )
-
-    @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def nix_collect_garbage(self) -> GenericJobMutationReturn:
-        job = start_nix_collect_garbage()
-
-        return GenericJobMutationReturn(
-            success=True,
-            code=200,
-            message="Garbage collector started...",
-            job=job_to_api_job(job),
-        )
--- a/selfprivacy_api/jobs/migrate_to_binds.py
+++ b/selfprivacy_api/jobs/migrate_to_binds.py
@ -67,8 +67,8 @@ def move_folder(

    try:
        data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
-    except Exception as error:
-        print(f"Error creating data path: {error}")
+    except Exception as e:
+        print(f"Error creating data path: {e}")
        return

    try:
--- a/selfprivacy_api/jobs/nix_collect_garbage.py
+++ b/selfprivacy_api/jobs/nix_collect_garbage.py
@ -1,147 +0,0 @@
-import re
-import subprocess
-from typing import Tuple, Iterable
-
-from selfprivacy_api.utils.huey import huey
-
-from selfprivacy_api.jobs import JobStatus, Jobs, Job
-
-
-class ShellException(Exception):
-    """Shell-related errors"""
-
-
-COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
-RESULT_WAS_NOT_FOUND_ERROR = (
-    "We are sorry, garbage collection result was not found. "
-    "Something went wrong, please report this to the support chat."
-)
-CLEAR_COMPLETED = "Garbage collection completed."
-
-
-def delete_old_gens_and_return_dead_report() -> str:
-    subprocess.run(
-        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
-        check=False,
-    )
-
-    result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
-        "utf-8"
-    )
-
-    return " " if result is None else result
-
-
-def run_nix_collect_garbage() -> Iterable[bytes]:
-    process = subprocess.Popen(
-        ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    )
-    return process.stdout if process.stdout else iter([])
-
-
-def parse_line(job: Job, line: str) -> Job:
-    """
-    We parse the string for the presence of a final line,
-    with the final amount of space cleared.
-    Simply put, we're just looking for a similar string:
-    "1537 store paths deleted, 339.84 MiB freed".
-    """
-    pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
-    match = re.search(pattern, line)
-
-    if match is None:
-        raise ShellException("nix returned gibberish output")
-
-    else:
-        Jobs.update(
-            job=job,
-            status=JobStatus.FINISHED,
-            status_text=CLEAR_COMPLETED,
-            result=f"{match.group(0)} have been cleared",
-        )
-    return job
-
-
-def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
-    completed_packages = 0
-    prev_progress = 0
-
-    for line in stream:
-        line = line.decode("utf-8")
-
-        if "deleting '/nix/store/" in line:
-            completed_packages += 1
-            percent = int((completed_packages / total_dead_packages) * 100)
-
-            if percent - prev_progress >= 5:
-                Jobs.update(
-                    job=job,
-                    status=JobStatus.RUNNING,
-                    progress=percent,
-                    status_text="Cleaning...",
-                )
-                prev_progress = percent
-
-        elif "store paths deleted," in line:
-            parse_line(job, line)
-
-
-def get_dead_packages(output) -> Tuple[int, float]:
-    dead = len(re.findall("/nix/store/", output))
-    percent = 0
-    if dead != 0:
-        percent = 100 / dead
-    return dead, percent
-
-
-@huey.task()
-def calculate_and_clear_dead_paths(job: Job):
-    Jobs.update(
-        job=job,
-        status=JobStatus.RUNNING,
-        progress=0,
-        status_text="Calculate the number of dead packages...",
-    )
-
-    dead_packages, package_equal_to_percent = get_dead_packages(
-        delete_old_gens_and_return_dead_report()
-    )
-
-    if dead_packages == 0:
-        Jobs.update(
-            job=job,
-            status=JobStatus.FINISHED,
-            status_text="Nothing to clear",
-            result="System is clear",
-        )
-        return True
-
-    Jobs.update(
-        job=job,
-        status=JobStatus.RUNNING,
-        progress=0,
-        status_text=f"Found {dead_packages} packages to remove!",
-    )
-
-    stream = run_nix_collect_garbage()
-    try:
-        process_stream(job, stream, dead_packages)
-    except ShellException as error:
-        Jobs.update(
-            job=job,
-            status=JobStatus.ERROR,
-            status_text=COMPLETED_WITH_ERROR,
-            error=RESULT_WAS_NOT_FOUND_ERROR,
-        )
-
-
-def start_nix_collect_garbage() -> Job:
-    job = Jobs.add(
-        type_id="maintenance.collect_nix_garbage",
-        name="Collect garbage",
-        description="Cleaning up unused packages",
-    )
-
-    calculate_and_clear_dead_paths(job=job)
-
-    return job
--- a/selfprivacy_api/task_registry.py
+++ b/selfprivacy_api/task_registry.py
@ -7,8 +7,6 @@ from selfprivacy_api.services.tasks import move_service
 from selfprivacy_api.jobs.upgrade_system import rebuild_system_task

 from selfprivacy_api.jobs.test import test_job
-from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths
-

 if environ.get("TEST_MODE"):
    from tests.test_huey import sum
--- a/setup.py
+++ b/setup.py
@ -2,7 +2,7 @@ from setuptools import setup, find_packages

 setup(
    name="selfprivacy_api",
-    version="3.2.0",
+    version="3.1.0",
    packages=find_packages(),
    scripts=[
        "selfprivacy_api/app.py",
--- a/tests/common.py
+++ b/tests/common.py
@ -2,8 +2,6 @@ import json
 from datetime import datetime, timezone, timedelta
 from mnemonic import Mnemonic

-from selfprivacy_api.jobs import Job, JobStatus
-
 # for expiration tests. If headache, consider freezegun
 RECOVERY_KEY_VALIDATION_DATETIME = "selfprivacy_api.models.tokens.time.datetime"
 DEVICE_KEY_VALIDATION_DATETIME = RECOVERY_KEY_VALIDATION_DATETIME
@ -81,12 +79,3 @@ def assert_recovery_recent(time_generated: str):
    assert datetime.fromisoformat(time_generated) - timedelta(seconds=5) < datetime.now(
        timezone.utc
    )
-
-
-def assert_job_errored(job: Job):
-    assert job is not None
-    assert job.status == JobStatus.ERROR
-
-    # consider adding a useful error message to an errored-out job
-    assert job.error is not None
-    assert job.error != ""
--- a/tests/test_backup.py
+++ b/tests/test_backup.py
@ -14,14 +14,13 @@ from selfprivacy_api.utils.huey import huey

 from selfprivacy_api.services.service import ServiceStatus

-from selfprivacy_api.graphql.queries.providers import BackupProvider as ProviderEnum
+from selfprivacy_api.graphql.queries.providers import BackupProvider
 from selfprivacy_api.graphql.common_types.backup import (
    RestoreStrategy,
    BackupReason,
 )
-from selfprivacy_api.graphql.queries.providers import BackupProvider

-from selfprivacy_api.jobs import Job, Jobs, JobStatus
+from selfprivacy_api.jobs import Jobs, JobStatus

 from selfprivacy_api.models.backup.snapshot import Snapshot

@ -39,10 +38,6 @@ from selfprivacy_api.backup.tasks import (
    reload_snapshot_cache,
 )
 from selfprivacy_api.backup.storage import Storage
-from selfprivacy_api.backup.local_secret import LocalBackupSecret
-from selfprivacy_api.backup.jobs import get_backup_fail
-
-from tests.common import assert_job_errored


 REPO_NAME = "test_backup"
@ -193,78 +188,6 @@ def test_backup_service(dummy_service, backups):
    assert_job_finished(f"services.{id}.backup", count=1)


-def all_job_text(job: Job) -> str:
-    # Use when we update to pydantic 2.xxx
-    # return Job.model_dump_json()
-    result = ""
-    if job.status_text is not None:
-        result += job.status_text
-    if job.description is not None:
-        result += job.description
-    if job.error is not None:
-        result += job.error
-
-    return result
-
-
-def test_error_censoring_encryptionkey(dummy_service, backups):
-    # Discard our key to inject a failure
-    old_key = LocalBackupSecret.get()
-    LocalBackupSecret.reset()
-    new_key = LocalBackupSecret.get()
-
-    with pytest.raises(ValueError):
-        # Should fail without correct key
-        Backups.back_up(dummy_service)
-
-    job = get_backup_fail(dummy_service)
-    assert_job_errored(job)
-
-    job_text = all_job_text(job)
-
-    assert old_key not in job_text
-    assert new_key not in job_text
-    # local backups do not have login key
-    # assert Backups.provider().key not in job_text
-
-    assert "CENSORED" in job_text
-
-
-def test_error_censoring_loginkey(dummy_service, backups, fp):
-    # We do not want to screw up our teardown
-    old_provider = Backups.provider()
-
-    secret = "aSecretNYA"
-
-    Backups.set_provider(
-        ProviderEnum.BACKBLAZE, login="meow", key=secret, location="moon"
-    )
-    assert Backups.provider().key == secret
-
-    # We could have called real backblaze but it is kind of not privacy so.
-    fp.allow_unregistered(True)
-    fp.register(
-        ["restic", fp.any()],
-        returncode=1,
-        stdout="only real cats are allowed",
-        # We do not want to suddenly call real backblaze even if code changes
-        occurrences=100,
-    )
-
-    with pytest.raises(ValueError):
-        Backups.back_up(dummy_service)
-
-    job = get_backup_fail(dummy_service)
-    assert_job_errored(job)
-
-    job_text = all_job_text(job)
-    assert secret not in job_text
-    assert job_text.count("CENSORED") == 2
-
-    # We do not want to screw up our teardown
-    Storage.store_provider(old_provider)
-
-
 def test_no_repo(memory_backup):
    with pytest.raises(ValueError):
        assert memory_backup.backupper.get_snapshots() == []
--- a/tests/test_graphql/test_nix_collect_garbage.py
+++ b/tests/test_graphql/test_nix_collect_garbage.py
@ -1,229 +0,0 @@
-# pylint: disable=redefined-outer-name
-# pylint: disable=unused-argument
-# pylint: disable=missing-function-docstring
-
-import pytest
-from selfprivacy_api.utils.huey import huey
-
-from selfprivacy_api.jobs import JobStatus, Jobs
-from tests.test_graphql.common import (
-    get_data,
-    assert_ok,
-    assert_empty,
-)
-
-from selfprivacy_api.jobs.nix_collect_garbage import (
-    get_dead_packages,
-    parse_line,
-    ShellException,
-)
-
-OUTPUT_PRINT_DEAD = """
-finding garbage collector roots...
-determining live/dead paths...
-/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7
-/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2
-/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21
-/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv
-/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook
-"""
-
-
-OUTPUT_COLLECT_GARBAGE = """
-removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
-finding garbage collector roots...
-deleting garbage...
-deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7'
-deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2'
-deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21'
-deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv'
-deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook'
-deleting unused links...
-note: currently hard linking saves -0.00 MiB
-190 store paths deleted, 425.51 MiB freed
-"""
-
-OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """
-removing old generations of profile /nix/var/nix/profiles/per-user/def/profile
-removing old generations of profile /nix/var/nix/profiles/per-user/def/channels
-finding garbage collector roots...
-deleting garbage...
-deleting unused links...
-note: currently hard linking saves 0.00 MiB
-0 store paths deleted, 0.00 MiB freed
-"""
-
-
-# ---
-
-
-def test_parse_line():
-    txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed"
-
-    job = Jobs.add(
-        name="name",
-        type_id="parse_line",
-        description="description",
-    )
-
-    output = parse_line(job, txt)
-    assert output.result == "425.51 MiB have been cleared"
-    assert output.status == JobStatus.FINISHED
-    assert output.error is None
-
-
-def test_parse_line_with_blank_line():
-    txt = ""
-    job = Jobs.add(
-        name="name",
-        type_id="parse_line",
-        description="description",
-    )
-
-    with pytest.raises(ShellException):
-        output = parse_line(job, txt)
-
-
-def test_get_dead_packages():
-    assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0)
-
-
-def test_get_dead_packages_zero():
-    assert get_dead_packages("") == (0, 0)
-
-
-RUN_NIX_COLLECT_GARBAGE_MUTATION = """
-mutation CollectGarbage {
-    system {
-        nixCollectGarbage {
-            success
-            message
-            code
-            job {
-                uid,
-                typeId,
-                name,
-                description,
-                status,
-                statusText,
-                progress,
-                createdAt,
-                updatedAt,
-                finishedAt,
-                error,
-                result,
-            }
-        }
-    }
-}
-"""
-
-
-def test_graphql_nix_collect_garbage(authorized_client, fp):
-    assert huey.immediate is True
-
-    fp.register(
-        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
-        stdout="",
-    )
-    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
-    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
-
-    response = authorized_client.post(
-        "/graphql",
-        json={
-            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
-        },
-    )
-
-    output = get_data(response)["system"]["nixCollectGarbage"]
-    assert_ok(output)
-    assert output["job"] is not None
-    assert output["job"]["status"] == "FINISHED"
-    assert output["job"]["error"] is None
-
-    assert (
-        fp.call_count(
-            [
-                "nix-env",
-                "-p",
-                "/nix/var/nix/profiles/system",
-                "--delete-generations old",
-            ]
-        )
-        == 1
-    )
-    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
-    assert fp.call_count(["nix-store", "--gc"]) == 1
-
-
-def test_graphql_nix_collect_garbage_return_zero_trash(authorized_client, fp):
-    assert huey.immediate is True
-
-    fp.register(
-        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
-        stdout="",
-    )
-    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
-    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH)
-
-    response = authorized_client.post(
-        "/graphql",
-        json={
-            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
-        },
-    )
-
-    output = get_data(response)["system"]["nixCollectGarbage"]
-    assert_ok(output)
-    assert output["job"] is not None
-    assert output["job"]["status"] == "FINISHED"
-    assert output["job"]["error"] is None
-
-    assert (
-        fp.call_count(
-            [
-                "nix-env",
-                "-p",
-                "/nix/var/nix/profiles/system",
-                "--delete-generations old",
-            ]
-        )
-        == 1
-    )
-    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 1
-    assert fp.call_count(["nix-store", "--gc"]) == 1
-
-
-def test_graphql_nix_collect_garbage_not_authorized_client(client, fp):
-    assert huey.immediate is True
-
-    fp.register(
-        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
-        stdout="",
-    )
-    fp.register(["nix-store", "--gc", "--print-dead"], stdout=OUTPUT_PRINT_DEAD)
-    fp.register(["nix-store", "--gc"], stdout=OUTPUT_COLLECT_GARBAGE)
-
-    response = client.post(
-        "/graphql",
-        json={
-            "query": RUN_NIX_COLLECT_GARBAGE_MUTATION,
-        },
-    )
-
-    assert_empty(response)
-
-    assert (
-        fp.call_count(
-            [
-                "nix-env",
-                "-p",
-                "/nix/var/nix/profiles/system",
-                "--delete-generations old",
-            ]
-        )
-        == 0
-    )
-    assert fp.call_count(["nix-store", "--gc", "--print-dead"]) == 0
-    assert fp.call_count(["nix-store", "--gc"]) == 0