diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/selfprivacy_api/graphql/mutations/system_mutations.py b/selfprivacy_api/graphql/mutations/system_mutations.py index 13ac16b..ee8ef15 100644 --- a/selfprivacy_api/graphql/mutations/system_mutations.py +++ b/selfprivacy_api/graphql/mutations/system_mutations.py @@ -6,9 +6,12 @@ from selfprivacy_api.graphql import IsAuthenticated from selfprivacy_api.graphql.mutations.mutation_interface import ( GenericMutationReturn, MutationReturnInterface, + GenericJobMutationReturn, ) import selfprivacy_api.actions.system as system_actions +from selfprivacy_api.graphql.common_types.jobs import job_to_api_job +from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage import selfprivacy_api.actions.ssh as ssh_actions @@ -191,3 +194,14 @@ class SystemMutations: message=f"Failed to pull repository changes:\n{result.data}", code=500, ) + + @strawberry.mutation(permission_classes=[IsAuthenticated]) + def nix_collect_garbage(self) -> GenericJobMutationReturn: + job = start_nix_collect_garbage() + + return GenericJobMutationReturn( + success=True, + code=200, + message="Garbage collector started...", + job=job_to_api_job(job), + ) diff --git a/selfprivacy_api/jobs/migrate_to_binds.py b/selfprivacy_api/jobs/migrate_to_binds.py index ce7229d..3250c9a 100644 --- a/selfprivacy_api/jobs/migrate_to_binds.py +++ b/selfprivacy_api/jobs/migrate_to_binds.py @@ -67,8 +67,8 @@ def move_folder( try: data_path.mkdir(mode=0o750, parents=True, exist_ok=True) - except Exception as e: - print(f"Error creating data path: {e}") + except Exception as error: + print(f"Error creating data path: {error}") return try: diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py new file mode 100644 index 0000000..1220fce --- /dev/null +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -0,0 +1,149 @@ +import re +import subprocess +from typing import Tuple, Iterable + +from selfprivacy_api.utils.huey import huey + +from selfprivacy_api.jobs import JobStatus, Jobs, Job + + +class ShellException(Exception): + """Shell-related errors""" + + +COMPLETED_WITH_ERROR = ( + "Error occurred, please report this to the support chat." +) +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found. " \ + "Something went wrong, please report this to the support chat." +CLEAR_COMPLETED = "Garbage collection completed." + + +def delete_old_gens_and_return_dead_report() -> str: + subprocess.run( + ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], + check=False, + ) + + result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( + "utf-8" + ) + + return " " if result is None else result + + +def run_nix_collect_garbage() -> Iterable[bytes]: + process = subprocess.Popen( + ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + return process.stdout if process.stdout else iter([]) + + +def parse_line(job: Job, line: str) -> Job: + """ + We parse the string for the presence of a final line, + with the final amount of space cleared. + Simply put, we're just looking for a similar string: + "1537 store paths deleted, 339.84 MiB freed". + """ + pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") + match = re.search(pattern, line) + + if match is None: + raise ShellException("nix returned gibberish output") + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + status_text=CLEAR_COMPLETED, + result=f"{match.group(0)} have been cleared", + ) + return job + + +def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None: + completed_packages = 0 + prev_progress = 0 + + for line in stream: + line = line.decode("utf-8") + + if "deleting '/nix/store/" in line: + completed_packages += 1 + percent = int((completed_packages / total_dead_packages) * 100) + + if percent - prev_progress >= 5: + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=percent, + status_text="Cleaning...", + ) + prev_progress = percent + + elif "store paths deleted," in line: + parse_line(job, line) + + +def get_dead_packages(output) -> Tuple[int, float]: + dead = len(re.findall("/nix/store/", output)) + percent = 0 + if dead != 0: + percent = 100 / dead + return dead, percent + + +@huey.task() +def calculate_and_clear_dead_paths(job: Job): + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text="Calculate the number of dead packages...", + ) + + dead_packages, package_equal_to_percent = get_dead_packages( + delete_old_gens_and_return_dead_report() + ) + + if dead_packages == 0: + + Jobs.update( + job=job, + status=JobStatus.FINISHED, + status_text="Nothing to clear", + result="System is clear", + ) + return True + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text=f"Found {dead_packages} packages to remove!", + ) + + stream = run_nix_collect_garbage() + try: + process_stream(job, stream, dead_packages) + except ShellException as error: + Jobs.update( + job=job, + status=JobStatus.ERROR, + status_text=COMPLETED_WITH_ERROR, + error=RESULT_WAS_NOT_FOUND_ERROR, + ) + + +def start_nix_collect_garbage() -> Job: + job = Jobs.add( + type_id="maintenance.collect_nix_garbage", + name="Collect garbage", + description="Cleaning up unused packages", + ) + + calculate_and_clear_dead_paths(job=job) + + return job diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index dfd329c..9c02cac 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,3 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.backup.tasks import * from selfprivacy_api.services.generic_service_mover import move_service +from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py new file mode 100644 index 0000000..6f00f79 --- /dev/null +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -0,0 +1,155 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=missing-function-docstring + +import pytest +from selfprivacy_api.utils.huey import huey + +from selfprivacy_api.jobs import JobStatus, Jobs, Job +from tests.test_graphql.common import get_data, assert_errorcode, assert_ok, assert_empty + +from selfprivacy_api.jobs.nix_collect_garbage import ( + get_dead_packages, + parse_line, + CLEAR_COMPLETED, + COMPLETED_WITH_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, + ShellException, +) + +OUTPUT_PRINT_DEAD = """ +finding garbage collector roots... +determining live/dead paths... +/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 +/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2 +/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21 +/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv +/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook +""" + + +OUTPUT_COLLECT_GARBAGE = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7' +deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2' +deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21' +deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv' +deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook' +deleting unused links... +note: currently hard linking saves -0.00 MiB +190 store paths deleted, 425.51 MiB freed +""" + +OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/profile +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting unused links... +note: currently hard linking saves 0.00 MiB +0 store paths deleted, 0.00 MiB freed +""" + +log_event = [] + + +@pytest.fixture +def mock_delete_old_gens_and_return_dead_report(mocker): + mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.delete_old_gens_and_return_dead_report", autospec=True, return_value=OUTPUT_PRINT_DEAD) + return mock + + +# --- + + +def test_parse_line(): + txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed" + + job = Jobs.add( + name="name", + type_id="parse_line", + description="description", + ) + + output = parse_line(job, txt) + assert output.result == '425.51 MiB have been cleared' + assert output.status == JobStatus.FINISHED + assert output.error is None + + +def test_parse_line_with_blank_line(): + txt = "" + job = Jobs.add( + name="name", + type_id="parse_line", + description="description", + ) + + with pytest.raises(ShellException): + output = parse_line(job, txt) + + + +def test_get_dead_packages(): + assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) + + +def test_get_dead_packages_zero(): + assert get_dead_packages("") == (0, 0) + + +RUN_NIX_COLLECT_GARBAGE_MUTATION = """ +mutation CollectGarbage { + system { + nixCollectGarbage { + success + message + code + job { + uid, + typeId, + name, + description, + status, + statusText, + progress, + createdAt, + updatedAt, + finishedAt, + error, + result, + } + } + } +} +""" + + +def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and_return_dead_report): + assert huey.immediate is True + response = authorized_client.post( + "/graphql", + json={ + "query": RUN_NIX_COLLECT_GARBAGE_MUTATION, + }, + ) + + output = get_data(response)["system"]["nixCollectGarbage"] + assert_ok(output) + assert output["job"] is not None + assert output["job"]["status"] == "FINISHED" + assert output["job"]["error"] is None + + +def test_graphql_nix_collect_garbage_not_authorized_client(client, mock_delete_old_gens_and_return_dead_report): + assert huey.immediate is True + response = client.post( + "/graphql", + json={ + "query": RUN_NIX_COLLECT_GARBAGE_MUTATION, + }, + ) + + assert_empty(response)