add nix-collect-garbage endpoint (#112)

Continuation of the broken #21

Co-authored-by: dettlaff <dettlaff@riseup.net>
Co-authored-by: def <dettlaff@riseup.net>
Co-authored-by: Houkime <>
Reviewed-on: SelfPrivacy/selfprivacy-rest-api#112
Reviewed-by: houkime <houkime@protonmail.com>

.drone.yml

@@ -5,13 +5,13 @@ name: default
 steps:
 - name: Run Tests and Generate Coverage Report
   commands:
-  - coverage run -m pytest -q
-  - coverage xml
+  - nix flake check -L
   - sonar-scanner -Dsonar.projectKey=SelfPrivacy-REST-API -Dsonar.sources=. -Dsonar.host.url=http://analyzer.lan:9000 -Dsonar.login="$SONARQUBE_TOKEN"
   environment:
     SONARQUBE_TOKEN:
       from_secret: SONARQUBE_TOKEN
 
+
 - name: Run Bandit Checks
   commands:
   - bandit -ll -r selfprivacy_api
@@ -22,3 +22,7 @@ steps:
 
 node:
   server: builder
+
+trigger:
+  event:
+    - push

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 80
+select = C,E,F,W,B,B950
+extend-ignore = E203, E501

.gitignore

@@ -147,3 +147,7 @@ cython_debug/
 # End of https://www.toptal.com/developers/gitignore/api/flask
 
 *.db
+*.rdb
+
+/result
+/.nixos-test-history

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/selfprivacy-rest-api.iml" filepath="$PROJECT_DIR$/.idea/selfprivacy-rest-api.iml" />
+    </modules>
+  </component>
+</project>

.idea/selfprivacy-rest-api.iml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.9" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CommitMessageInspectionProfile">
+    <profile version="1.0">
+      <inspection_tool class="CommitFormat" enabled="true" level="WARNING" enabled_by_default="true" />
+      <inspection_tool class="CommitNamingConvention" enabled="true" level="WARNING" enabled_by_default="true" />
+    </profile>
+  </component>
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+plugins = pydantic.mypy

.pylintrc

@@ -1,3 +1,6 @@
 [MASTER]
 init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))"
 extension-pkg-whitelist=pydantic
+
+[FORMAT]
+max-line-length=88

CONTRIBUTING.md

@@ -0,0 +1,88 @@
+# SelfPrivacy API contributors guide
+
+Instructions for [VScode](https://code.visualstudio.com) or [VScodium](https://github.com/VSCodium/vscodium) under Unix-like platform.
+
+1. **To get started, create an account for yourself on the** [**SelfPrivacy Gitea**](https://git.selfprivacy.org/user/sign_up). Proceed to fork
+the [repository](https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api), and clone it on your local computer:
+
+    ```git clone https://git.selfprivacy.org/your_user_name/selfprivacy-rest-api```
+
+2. **Install Nix**
+
+    ```sh <(curl -L https://nixos.org/nix/install)```
+
+    For detailed installation information, please review and follow: [link](https://nixos.org/manual/nix/stable/installation/installing-binary.html#installing-a-binary-distribution).
+
+3. **Change directory to the cloned repository and start a nix shell:**
+
+    ```cd selfprivacy-rest-api && nix-shell```
+
+    Nix will install all of the necessary packages for development work, all further actions will take place only within nix-shell.
+
+4. **Install these plugins for VScode/VScodium**
+
+    Required: ```ms-python.python```, ```ms-python.vscode-pylance```
+
+    Optional, but highly recommended: ```ms-python.black-formatter```, ```bbenoist.Nix```, ```ryanluker.vscode-coverage-gutters```
+
+5. **Set the path to the python interpreter from the nix store.** To do this, execute the command:
+
+    ```whereis python```
+
+    Copy the path that starts with ```/nix/store/``` and ends with ```env/bin/python```
+
+    ```/nix/store/???-python3-3.9.??-env/bin/python```
+
+    Click on the python version selection in the lower right corner, and replace the path to the interpreter in the project with the one you copied from the terminal.
+
+6. **Congratulations :) Now you can develop new changes and test the project locally in a Nix environment.**
+
+## What do you need to know before starting development work?
+- RestAPI is no longer utilized, the project has moved to [GraphQL](https://graphql.org), however, the API functionality still works on Rest
+
+
+## What to do after making changes to the repository?
+
+**Run unit tests** using ```pytest .```
+Make sure that all tests pass successfully and the API works correctly. For convenience, you can use the built-in VScode interface.
+
+How to review the percentage of code coverage? Execute the command:
+
+```coverage run -m pytest && coverage xml && coverage report```
+
+Next, use the recommended extension ```ryanluker.vscode-coverage-gutters```, navigate to one of the test files, and click the "watch" button on the bottom panel of VScode.
+
+**Format (linting) code**, we use [black](https://pypi.org/project/black/) formatting, enter
+```black .``` to automatically format files, or use the recommended extension.
+
+**And please remember, we have adopted** [**commit naming convention**](https://www.conventionalcommits.org/en/v1.0.0/), follow the link for more information.
+
+Please request a review from at least one of the other maintainers. If you are not sure who to request, request a review from SelfPrivacy/Devs team.
+
+## Helpful links!
+
+**SelfPrivacy Contributor chat :3**
+
+- [**Telegram:** @selfprivacy_dev](https://t.me/selfprivacy_dev)
+- [**Matrix:** #dev:selfprivacy.org](https://matrix.to/#/#dev:selfprivacy.org)
+
+**Helpful material to review:**
+
+- [GraphQL Query Language Documentation](https://graphql.org/)
+- [Documentation Strawberry - python library for working with GraphQL](https://strawberry.rocks/docs/)
+- [Nix Documentation](https://nixos.org/guides/ad-hoc-developer-environments.html)
+
+### Track your time
+
+If you are working on a task, please track your time and add it to the commit message. For example:
+
+```
+feat: add new feature
+
+- did some work
+- did some more work
+
+fixes #4, spent @1h30m
+```
+
+[Timewarrior](https://timewarrior.net/) is a good tool for tracking time.

README.md

@@ -0,0 +1,92 @@
+# SelfPrivacy GraphQL API which allows app to control your server
+
+![CI status](https://ci.selfprivacy.org/api/badges/SelfPrivacy/selfprivacy-rest-api/status.svg)
+
+## Build
+
+```console
+$ nix build
+```
+
+In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
+
+## Develop
+
+```console
+$ nix develop
+[SP devshell:/dir/selfprivacy-rest-api]$ python
+Python 3.10.13 (main, Aug 24 2023, 12:59:26) [GCC 12.3.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+(ins)>>>
+```
+
+If you don't have experimental flakes enabled, you can use the following command:
+
+```console
+$ nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
+```
+
+## Testing
+
+Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
+```console
+$ nix flake check -L
+```
+
+Run the same test suite, but additionally create `./result/coverage.xml` in the current directory:
+```console
+$ nix build .#checks.x86_64-linux.default -L
+```
+
+Alternatively, just print the path to `/nix/store/...coverage.xml` without creating any files in the current directory:
+```console
+$ nix build .#checks.x86_64-linux.default -L --print-out-paths --no-link
+```
+
+Run the same test suite with arbitrary pytest options:
+```console
+$ pytest-vm.sh # specify pytest options here, e.g. `--last-failed`
+```
+When running using the script, pytest cache is preserved between runs in `.pytest_cache` folder.
+NixOS VM state temporary resides in `${TMPDIR:=/tmp}/nixos-vm-tmp-dir/vm-state-machine` during the test.
+Git workdir directory is shared read-write with VM via `.nixos-vm-tmp-dir/shared-xchg` symlink. VM accesses workdir contents via `/tmp/shared` mount point and `/root/source` symlink.
+
+Launch VM and execute commands manually either in Linux console (user `root`) or using python NixOS tests driver API (refer to [NixOS documentation](https://nixos.org/manual/nixos/stable/#ssec-machine-objects)):
+```console
+$ nix run .#checks.x86_64-linux.default.driverInteractive
+```
+
+You can add `--keep-vm-state` in order to keep VM state between runs:
+```console
+$ TMPDIR=".nixos-vm-tmp-dir" nix run .#checks.x86_64-linux.default.driverInteractive --keep-vm-state
+```
+
+Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.
+
+## Dependencies and Dependant Modules
+
+This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.
+
+In order to synchronize nixpkgs input with the same from selfprivacy-nixos-config repository, use this command:
+
+```console
+$ nix flake lock --override-input nixpkgs nixpkgs --inputs-from git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
+```
+
+Replace BRANCH with the branch name of selfprivacy-nixos-config repository you want to sync with. During development nixpkgs input update might be required in both selfprivacy-rest-api and selfprivacy-nixos-config repositories simultaneously. So, a new feature branch might be temporarily used until selfprivacy-nixos-config gets the feature branch merged.
+
+Show current flake inputs (e.g. nixpkgs):
+```console
+$ nix flake metadata
+```
+
+Show selfprivacy-nixos-config Nix flake inputs (including nixpkgs):
+```console
+$ nix flake metadata git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
+```
+
+Nix code for NixOS service module for API is located in NixOS configuration repository.
+
+## Troubleshooting
+
+Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.

api.nix

@@ -1,64 +0,0 @@
-{ lib, python39Packages }:
-with python39Packages;
-buildPythonApplication {
-  pname = "selfprivacy-api";
-  version = "2.0.0";
-
-  propagatedBuildInputs = [
-    setuptools
-        portalocker
-        pytz
-        pytest
-        pytest-mock
-        pytest-datadir
-        huey
-        gevent
-        mnemonic
-        pydantic
-        typing-extensions
-        psutil
-        fastapi
-        uvicorn
-        (buildPythonPackage rec {
-          pname = "strawberry-graphql";
-          version = "0.123.0";
-          format = "pyproject";
-          patches = [
-            ./strawberry-graphql.patch
-          ];
-          propagatedBuildInputs = [
-            typing-extensions
-            python-multipart
-            python-dateutil
-            # flask
-            pydantic
-            pygments
-            poetry
-            # flask-cors
-            (buildPythonPackage rec {
-              pname = "graphql-core";
-              version = "3.2.0";
-              format = "setuptools";
-              src = fetchPypi {
-                inherit pname version;
-                sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84=";
-              };
-              checkInputs = [
-                pytest-asyncio
-                pytest-benchmark
-                pytestCheckHook
-              ];
-              pythonImportsCheck = [
-                "graphql"
-              ];
-            })
-          ];
-          src = fetchPypi {
-            inherit pname version;
-            sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo=";
-          };
-        })
-      ];
-
-  src = ./.;
-}

default.nix

@@ -1,2 +1,29 @@
-{ pkgs ? import <nixpkgs> {} }:
-pkgs.callPackage ./api.nix {}
+{ pythonPackages, rev ? "local" }:
+
+pythonPackages.buildPythonPackage rec {
+  pname = "selfprivacy-graphql-api";
+  version = rev;
+  src = builtins.filterSource (p: t: p != ".git" && t != "symlink") ./.;
+  propagatedBuildInputs = with pythonPackages; [
+    fastapi
+    gevent
+    huey
+    mnemonic
+    portalocker
+    psutil
+    pydantic
+    pytz
+    redis
+    setuptools
+    strawberry-graphql
+    typing-extensions
+    uvicorn
+  ];
+  pythonImportsCheck = [ "selfprivacy_api" ];
+  doCheck = false;
+  meta = {
+    description = ''
+      SelfPrivacy Server Management API
+    '';
+  };
+}

flake.lock

@@ -0,0 +1,26 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1709677081,
+        "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,162 @@
+{
+  description = "SelfPrivacy API flake";
+
+  inputs.nixpkgs.url = "github:nixos/nixpkgs";
+
+  outputs = { self, nixpkgs, ... }:
+    let
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+      selfprivacy-graphql-api = pkgs.callPackage ./default.nix {
+        pythonPackages = pkgs.python310Packages;
+        rev = self.shortRev or self.dirtyShortRev or "dirty";
+      };
+      python = self.packages.${system}.default.pythonModule;
+      python-env =
+        python.withPackages (ps:
+          self.packages.${system}.default.propagatedBuildInputs ++ (with ps; [
+            coverage
+            pytest
+            pytest-datadir
+            pytest-mock
+            pytest-subprocess
+            black
+            mypy
+            pylsp-mypy
+            python-lsp-black
+            python-lsp-server
+            pyflakes
+            typer # for strawberry
+            types-redis # for mypy
+          ] ++ strawberry-graphql.optional-dependencies.cli));
+
+      vmtest-src-dir = "/root/source";
+      shellMOTD = ''
+        Welcome to SP API development shell!
+
+        [formatters]
+
+          black
+          nixpkgs-fmt
+
+        [testing in NixOS VM]
+
+          nixos-test-driver - run an interactive NixOS VM with all dependencies included and 2 disk volumes
+          pytest-vm         - run pytest in an ephemeral NixOS VM with Redis, accepting pytest arguments
+      '';
+    in
+    {
+      # see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
+      packages.${system} = {
+        default = selfprivacy-graphql-api;
+        pytest-vm = pkgs.writeShellScriptBin "pytest-vm" ''
+          set -o errexit
+          set -o nounset
+          set -o xtrace
+
+          # see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
+          export TMPDIR=''${TMPDIR:=/tmp}/nixos-vm-tmp-dir
+          readonly NIXOS_VM_SHARED_DIR_HOST="$TMPDIR/shared-xchg"
+          readonly NIXOS_VM_SHARED_DIR_GUEST="/tmp/shared"
+
+          mkdir -p "$TMPDIR"
+          ln -sfv "$PWD" -T "$NIXOS_VM_SHARED_DIR_HOST"
+
+          SCRIPT=$(cat <<EOF
+          start_all()
+          machine.succeed("ln -sf $NIXOS_VM_SHARED_DIR_GUEST -T ${vmtest-src-dir} >&2")
+          machine.succeed("cd ${vmtest-src-dir} && coverage run -m pytest -v $@ >&2")
+          machine.succeed("cd ${vmtest-src-dir} && coverage report >&2")
+          EOF
+          )
+
+          if [ -f "/etc/arch-release" ]; then
+              ${self.checks.${system}.default.driverInteractive}/bin/nixos-test-driver --no-interactive <(printf "%s" "$SCRIPT")
+          else
+              ${self.checks.${system}.default.driver}/bin/nixos-test-driver -- <(printf "%s" "$SCRIPT")
+          fi
+        '';
+      };
+      nixosModules.default =
+        import ./nixos/module.nix self.packages.${system}.default;
+      devShells.${system}.default = pkgs.mkShellNoCC {
+        name = "SP API dev shell";
+        packages = with pkgs; [
+          nixpkgs-fmt
+          rclone
+          redis
+          restic
+          self.packages.${system}.pytest-vm
+          # FIXME consider loading this explicitly only after ArchLinux issue is solved
+          self.checks.x86_64-linux.default.driverInteractive
+          # the target API application python environment
+          python-env
+        ];
+        shellHook = ''
+          # envs set with export and as attributes are treated differently.
+          # for example. printenv <Name> will not fetch the value of an attribute.
+          export TEST_MODE="true"
+
+          # more tips for bash-completion to work on non-NixOS:
+          # https://discourse.nixos.org/t/whats-the-nix-way-of-bash-completion-for-packages/20209/16?u=alexoundos
+          # Load installed profiles
+          for file in "/etc/profile.d/"*.sh; do
+            # If that folder doesn't exist, bash loves to return the whole glob
+            [[ -f "$file" ]] && source "$file"
+          done
+
+          printf "%s" "${shellMOTD}"
+        '';
+      };
+      checks.${system} = {
+        fmt-check = pkgs.runCommandLocal "sp-api-fmt-check"
+          { nativeBuildInputs = [ pkgs.black ]; }
+          "black --check ${self.outPath} > $out";
+        default =
+          pkgs.testers.runNixOSTest {
+            name = "default";
+            nodes.machine = { lib, pkgs, ... }: {
+              # 2 additional disks (1024 MiB and 200 MiB) with empty ext4 FS
+              virtualisation.emptyDiskImages = [ 1024 200 ];
+              virtualisation.fileSystems."/volumes/vdb" = {
+                autoFormat = true;
+                device = "/dev/vdb"; # this name is chosen by QEMU, not here
+                fsType = "ext4";
+                noCheck = true;
+              };
+              virtualisation.fileSystems."/volumes/vdc" = {
+                autoFormat = true;
+                device = "/dev/vdc"; # this name is chosen by QEMU, not here
+                fsType = "ext4";
+                noCheck = true;
+              };
+              boot.consoleLogLevel = lib.mkForce 3;
+              documentation.enable = false;
+              services.journald.extraConfig = lib.mkForce "";
+              services.redis.servers.sp-api = {
+                enable = true;
+                save = [ ];
+                settings.notify-keyspace-events = "KEA";
+              };
+              environment.systemPackages = with pkgs; [
+                python-env
+                # TODO: these can be passed via wrapper script around app
+                rclone
+                restic
+              ];
+              environment.variables.TEST_MODE = "true";
+              systemd.tmpfiles.settings.src.${vmtest-src-dir}.L.argument =
+                self.outPath;
+            };
+            testScript = ''
+              start_all()
+              machine.succeed("cd ${vmtest-src-dir} && coverage run --data-file=/tmp/.coverage -m pytest -p no:cacheprovider -v >&2")
+              machine.succeed("coverage xml --rcfile=${vmtest-src-dir}/.coveragerc --data-file=/tmp/.coverage >&2")
+              machine.copy_from_vm("coverage.xml", ".")
+              machine.succeed("coverage report >&2")
+            '';
+          };
+      };
+    };
+  nixConfig.bash-prompt = ''\n\[\e[1;32m\][\[\e[0m\]\[\e[1;34m\]SP devshell\[\e[0m\]\[\e[1;32m\]:\w]\$\[\[\e[0m\] '';
+}

nix-dependencies-diagram.puml

@@ -0,0 +1,22 @@
+@startuml
+
+left to right direction
+
+title repositories and flake inputs relations diagram
+
+cloud nixpkgs as nixpkgs_transit
+control "<font:monospaced><size:15>nixos-rebuild" as nixos_rebuild
+component "SelfPrivacy\nAPI app" as selfprivacy_app
+component "SelfPrivacy\nNixOS configuration" as nixos_configuration
+
+note top of nixos_configuration : SelfPrivacy\nAPI service module
+
+nixos_configuration ).. nixpkgs_transit
+nixpkgs_transit ..> selfprivacy_app
+selfprivacy_app --> nixos_configuration
+[nixpkgs] --> nixos_configuration
+nixos_configuration -> nixos_rebuild
+
+footer %date("yyyy-MM-dd'T'HH:mmZ")
+
+@enduml

nixos/module.nix

@@ -0,0 +1,166 @@
+selfprivacy-graphql-api: { config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.selfprivacy-api;
+  config-id = "default";
+  nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+  nix = "${config.nix.package.out}/bin/nix";
+in
+{
+  options.services.selfprivacy-api = {
+    enable = lib.mkOption {
+      default = true;
+      type = lib.types.bool;
+      description = ''
+        Enable SelfPrivacy API service
+      '';
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    users.users."selfprivacy-api" = {
+      isNormalUser = false;
+      isSystemUser = true;
+      extraGroups = [ "opendkim" ];
+      group = "selfprivacy-api";
+    };
+    users.groups."selfprivacy-api".members = [ "selfprivacy-api" ];
+
+    systemd.services.selfprivacy-api = {
+      description = "API Server used to control system from the mobile application";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+        PYTHONUNBUFFERED = "1";
+      } // config.networking.proxy.envVars;
+      path = [
+        "/var/"
+        "/var/dkim/"
+        pkgs.coreutils
+        pkgs.gnutar
+        pkgs.xz.bin
+        pkgs.gzip
+        pkgs.gitMinimal
+        config.nix.package.out
+        pkgs.restic
+        pkgs.mkpasswd
+        pkgs.util-linux
+        pkgs.e2fsprogs
+        pkgs.iproute2
+      ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "network-online.target" ];
+      serviceConfig = {
+        User = "root";
+        ExecStart = "${selfprivacy-graphql-api}/bin/app.py";
+        Restart = "always";
+        RestartSec = "5";
+      };
+    };
+    systemd.services.selfprivacy-api-worker = {
+      description = "Task worker for SelfPrivacy API";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+        PYTHONUNBUFFERED = "1";
+        PYTHONPATH =
+          pkgs.python310Packages.makePythonPath [ selfprivacy-graphql-api ];
+      } // config.networking.proxy.envVars;
+      path = [
+        "/var/"
+        "/var/dkim/"
+        pkgs.coreutils
+        pkgs.gnutar
+        pkgs.xz.bin
+        pkgs.gzip
+        pkgs.gitMinimal
+        config.nix.package.out
+        pkgs.restic
+        pkgs.mkpasswd
+        pkgs.util-linux
+        pkgs.e2fsprogs
+        pkgs.iproute2
+      ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "network-online.target" ];
+      serviceConfig = {
+        User = "root";
+        ExecStart = "${pkgs.python310Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey";
+        Restart = "always";
+        RestartSec = "5";
+      };
+    };
+    # One shot systemd service to rebuild NixOS using nixos-rebuild
+    systemd.services.sp-nixos-rebuild = {
+      description = "nixos-rebuild switch";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      # TODO set proper timeout for reboot instead of service restart
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        # sync top-level flake with sp-modules sub-flake
+        # (https://github.com/NixOS/nix/issues/9339)
+        ExecStartPre = ''
+          ${nix} flake lock --override-input sp-modules path:./sp-modules
+        '';
+        ExecStart = ''
+          ${nixos-rebuild} switch --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+    # One shot systemd service to upgrade NixOS using nixos-rebuild
+    systemd.services.sp-nixos-upgrade = {
+      # protection against simultaneous runs
+      after = [ "sp-nixos-rebuild.service" ];
+      description = "Upgrade NixOS and SP modules to latest versions";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        # TODO get URL from systemd template parameter?
+        ExecStartPre = ''
+          ${nix} flake update \
+          --override-input selfprivacy-nixos-config git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes
+        '';
+        ExecStart = ''
+          ${nixos-rebuild} switch --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+    # One shot systemd service to rollback NixOS using nixos-rebuild
+    systemd.services.sp-nixos-rollback = {
+      # protection against simultaneous runs
+      after = [ "sp-nixos-rebuild.service" "sp-nixos-upgrade.service" ];
+      description = "Rollback NixOS using nixos-rebuild";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        ExecStart = ''
+          ${nixos-rebuild} switch --rollback --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+  };
+}

selfprivacy_api/actions/api_tokens.py

@@ -1,21 +1,24 @@
-"""App tokens actions"""
-from datetime import datetime
+"""
+App tokens actions. 
+The only actions on tokens that are accessible from APIs
+"""
+from datetime import datetime, timezone
 from typing import Optional
 from pydantic import BaseModel
+from mnemonic import Mnemonic
 
-
-from selfprivacy_api.utils.auth import (
-    delete_token,
-    generate_recovery_token,
-    get_recovery_token_status,
-    get_tokens_info,
-    is_recovery_token_exists,
-    is_recovery_token_valid,
-    is_token_name_exists,
-    is_token_name_pair_valid,
-    refresh_token,
-    get_token_name,
+from selfprivacy_api.utils.timeutils import ensure_tz_aware, ensure_tz_aware_strict
+from selfprivacy_api.repositories.tokens.redis_tokens_repository import (
+    RedisTokensRepository,
 )
+from selfprivacy_api.repositories.tokens.exceptions import (
+    TokenNotFound,
+    RecoveryKeyNotFound,
+    InvalidMnemonic,
+    NewDeviceKeyNotFound,
+)
+
+TOKEN_REPO = RedisTokensRepository()
 
 
 class TokenInfoWithIsCaller(BaseModel):
@@ -26,20 +29,33 @@ class TokenInfoWithIsCaller(BaseModel):
     is_caller: bool
 
 
+def _naive(date_time: datetime) -> datetime:
+    if date_time is None:
+        return None
+    if date_time.tzinfo is not None:
+        date_time.astimezone(timezone.utc)
+    return date_time.replace(tzinfo=None)
+
+
 def get_api_tokens_with_caller_flag(caller_token: str) -> list[TokenInfoWithIsCaller]:
     """Get the tokens info"""
-    caller_name = get_token_name(caller_token)
-    tokens = get_tokens_info()
+    caller_name = TOKEN_REPO.get_token_by_token_string(caller_token).device_name
+    tokens = TOKEN_REPO.get_tokens()
     return [
         TokenInfoWithIsCaller(
-            name=token.name,
-            date=token.date,
-            is_caller=token.name == caller_name,
+            name=token.device_name,
+            date=token.created_at,
+            is_caller=token.device_name == caller_name,
         )
         for token in tokens
     ]
 
 
+def is_token_valid(token) -> bool:
+    """Check if token is valid"""
+    return TOKEN_REPO.is_token_valid(token)
+
+
 class NotFoundException(Exception):
     """Not found exception"""
 
@@ -50,19 +66,22 @@ class CannotDeleteCallerException(Exception):
 
 def delete_api_token(caller_token: str, token_name: str) -> None:
     """Delete the token"""
-    if is_token_name_pair_valid(token_name, caller_token):
+    if TOKEN_REPO.is_token_name_pair_valid(token_name, caller_token):
         raise CannotDeleteCallerException("Cannot delete caller's token")
-    if not is_token_name_exists(token_name):
+    if not TOKEN_REPO.is_token_name_exists(token_name):
         raise NotFoundException("Token not found")
-    delete_token(token_name)
+    token = TOKEN_REPO.get_token_by_name(token_name)
+    TOKEN_REPO.delete_token(token)
 
 
 def refresh_api_token(caller_token: str) -> str:
     """Refresh the token"""
-    new_token = refresh_token(caller_token)
-    if new_token is None:
+    try:
+        old_token = TOKEN_REPO.get_token_by_token_string(caller_token)
+        new_token = TOKEN_REPO.refresh_token(old_token)
+    except TokenNotFound:
         raise NotFoundException("Token not found")
-    return new_token
+    return new_token.token
 
 
 class RecoveryTokenStatus(BaseModel):
@@ -76,19 +95,23 @@ class RecoveryTokenStatus(BaseModel):
 
 
 def get_api_recovery_token_status() -> RecoveryTokenStatus:
-    """Get the recovery token status"""
-    if not is_recovery_token_exists():
+    """Get the recovery token status, timezone-aware"""
+    token = TOKEN_REPO.get_recovery_key()
+    if token is None:
         return RecoveryTokenStatus(exists=False, valid=False)
-    status = get_recovery_token_status()
-    if status is None:
-        return RecoveryTokenStatus(exists=False, valid=False)
-    is_valid = is_recovery_token_valid()
+    is_valid = TOKEN_REPO.is_recovery_key_valid()
+
+    # New tokens are tz-aware, but older ones might not be
+    expiry_date = token.expires_at
+    if expiry_date is not None:
+        expiry_date = ensure_tz_aware_strict(expiry_date)
+
     return RecoveryTokenStatus(
         exists=True,
         valid=is_valid,
-        date=status["date"],
-        expiration=status["expiration"],
-        uses_left=status["uses_left"],
+        date=ensure_tz_aware_strict(token.created_at),
+        expiration=expiry_date,
+        uses_left=token.uses_left,
     )
 
 
@@ -105,12 +128,54 @@ def get_new_api_recovery_key(
 ) -> str:
     """Get new recovery key"""
     if expiration_date is not None:
-        current_time = datetime.now().timestamp()
-        if expiration_date.timestamp() < current_time:
+        expiration_date = ensure_tz_aware(expiration_date)
+        current_time = datetime.now(timezone.utc)
+        if expiration_date < current_time:
             raise InvalidExpirationDate("Expiration date is in the past")
     if uses_left is not None:
         if uses_left <= 0:
             raise InvalidUsesLeft("Uses must be greater than 0")
 
-    key = generate_recovery_token(expiration_date, uses_left)
-    return key
+    key = TOKEN_REPO.create_recovery_key(expiration_date, uses_left)
+    mnemonic_phrase = Mnemonic(language="english").to_mnemonic(bytes.fromhex(key.key))
+    return mnemonic_phrase
+
+
+def use_mnemonic_recovery_token(mnemonic_phrase, name):
+    """Use the recovery token by converting the mnemonic word list to a byte array.
+    If the recovery token if invalid itself, return None
+    If the binary representation of phrase not matches
+    the byte array of the recovery token, return None.
+    If the mnemonic phrase is valid then generate a device token and return it.
+    Substract 1 from uses_left if it exists.
+    mnemonic_phrase is a string representation of the mnemonic word list.
+    """
+    try:
+        token = TOKEN_REPO.use_mnemonic_recovery_key(mnemonic_phrase, name)
+        return token.token
+    except (RecoveryKeyNotFound, InvalidMnemonic):
+        return None
+
+
+def delete_new_device_auth_token() -> None:
+    TOKEN_REPO.delete_new_device_key()
+
+
+def get_new_device_auth_token() -> str:
+    """Generate and store a new device auth token which is valid for 10 minutes
+    and return a mnemonic phrase representation
+    """
+    key = TOKEN_REPO.get_new_device_key()
+    return Mnemonic(language="english").to_mnemonic(bytes.fromhex(key.key))
+
+
+def use_new_device_auth_token(mnemonic_phrase, name) -> Optional[str]:
+    """Use the new device auth token by converting the mnemonic string to a byte array.
+    If the mnemonic phrase is valid then generate a device token and return it.
+    New device auth token must be deleted.
+    """
+    try:
+        token = TOKEN_REPO.use_mnemonic_new_device_key(mnemonic_phrase, name)
+        return token.token
+    except (NewDeviceKeyNotFound, InvalidMnemonic):
+        return None

selfprivacy_api/actions/services.py

@@ -0,0 +1,34 @@
+from selfprivacy_api.utils.block_devices import BlockDevices
+from selfprivacy_api.jobs import Jobs, Job
+
+from selfprivacy_api.services import get_service_by_id
+from selfprivacy_api.services.tasks import move_service as move_service_task
+
+
+class ServiceNotFoundError(Exception):
+    pass
+
+
+class VolumeNotFoundError(Exception):
+    pass
+
+
+def move_service(service_id: str, volume_name: str) -> Job:
+    service = get_service_by_id(service_id)
+    if service is None:
+        raise ServiceNotFoundError(f"No such service:{service_id}")
+
+    volume = BlockDevices().get_block_device(volume_name)
+    if volume is None:
+        raise VolumeNotFoundError(f"No such volume:{volume_name}")
+
+    service.assert_can_move(volume)
+
+    job = Jobs.add(
+        type_id=f"services.{service.get_id()}.move",
+        name=f"Move {service.get_display_name()}",
+        description=f"Moving {service.get_display_name()} data to {volume.name}",
+    )
+
+    move_service_task(service, volume, job)
+    return job

selfprivacy_api/actions/ssh.py

@@ -31,7 +31,7 @@ def get_ssh_settings() -> UserdataSshSettings:
         if "enable" not in data["ssh"]:
             data["ssh"]["enable"] = True
         if "passwordAuthentication" not in data["ssh"]:
-            data["ssh"]["passwordAuthentication"] = True
+            data["ssh"]["passwordAuthentication"] = False
         if "rootKeys" not in data["ssh"]:
             data["ssh"]["rootKeys"] = []
         return UserdataSshSettings(**data["ssh"])
@@ -49,19 +49,6 @@ def set_ssh_settings(
             data["ssh"]["passwordAuthentication"] = password_authentication
 
 
-def add_root_ssh_key(public_key: str):
-    with WriteUserData() as data:
-        if "ssh" not in data:
-            data["ssh"] = {}
-        if "rootKeys" not in data["ssh"]:
-            data["ssh"]["rootKeys"] = []
-        # Return 409 if key already in array
-        for key in data["ssh"]["rootKeys"]:
-            if key == public_key:
-                raise KeyAlreadyExists()
-        data["ssh"]["rootKeys"].append(public_key)
-
-
 class KeyAlreadyExists(Exception):
     """Key already exists"""
 

selfprivacy_api/actions/system.py

@@ -2,8 +2,10 @@
 import os
 import subprocess
 import pytz
-from typing import Optional
+from typing import Optional, List
 from pydantic import BaseModel
+from selfprivacy_api.jobs import Job, JobStatus, Jobs
+from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
 
 from selfprivacy_api.utils import WriteUserData, ReadUserData
 
@@ -13,7 +15,7 @@ def get_timezone() -> str:
     with ReadUserData() as user_data:
         if "timezone" in user_data:
             return user_data["timezone"]
-        return "Europe/Uzhgorod"
+        return "Etc/UTC"
 
 
 class InvalidTimezone(Exception):
@@ -58,36 +60,68 @@ def set_auto_upgrade_settings(
             user_data["autoUpgrade"]["allowReboot"] = allowReboot
 
 
-def rebuild_system() -> int:
-    """Rebuild the system"""
-    rebuild_result = subprocess.Popen(
-        ["systemctl", "start", "sp-nixos-rebuild.service"], start_new_session=True
+class ShellException(Exception):
+    """Something went wrong when calling another process"""
+
+    pass
+
+
+def run_blocking(cmd: List[str], new_session: bool = False) -> str:
+    """Run a process, block until done, return output, complain if failed"""
+    process_handle = subprocess.Popen(
+        cmd,
+        shell=False,
+        start_new_session=new_session,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
     )
-    rebuild_result.communicate()[0]
-    return rebuild_result.returncode
+    stdout_raw, stderr_raw = process_handle.communicate()
+    stdout = stdout_raw.decode("utf-8")
+    if stderr_raw is not None:
+        stderr = stderr_raw.decode("utf-8")
+    else:
+        stderr = ""
+    output = stdout + "\n" + stderr
+    if process_handle.returncode != 0:
+        raise ShellException(
+            f"Shell command failed, command array: {cmd}, output: {output}"
+        )
+    return stdout
+
+
+def rebuild_system() -> Job:
+    """Rebuild the system"""
+    job = Jobs.add(
+        type_id="system.nixos.rebuild",
+        name="Rebuild system",
+        description="Applying the new system configuration by building the new NixOS generation.",
+        status=JobStatus.CREATED,
+    )
+    rebuild_system_task(job)
+    return job
 
 
 def rollback_system() -> int:
     """Rollback the system"""
-    rollback_result = subprocess.Popen(
-        ["systemctl", "start", "sp-nixos-rollback.service"], start_new_session=True
-    )
-    rollback_result.communicate()[0]
-    return rollback_result.returncode
+    run_blocking(["systemctl", "start", "sp-nixos-rollback.service"], new_session=True)
+    return 0
 
 
-def upgrade_system() -> int:
+def upgrade_system() -> Job:
     """Upgrade the system"""
-    upgrade_result = subprocess.Popen(
-        ["systemctl", "start", "sp-nixos-upgrade.service"], start_new_session=True
+    job = Jobs.add(
+        type_id="system.nixos.upgrade",
+        name="Upgrade system",
+        description="Upgrading the system to the latest version.",
+        status=JobStatus.CREATED,
     )
-    upgrade_result.communicate()[0]
-    return upgrade_result.returncode
+    rebuild_system_task(job, upgrade=True)
+    return job
 
 
 def reboot_system() -> None:
     """Reboot the system"""
-    subprocess.Popen(["reboot"], start_new_session=True)
+    run_blocking(["reboot"], new_session=True)
 
 
 def get_system_version() -> str:

selfprivacy_api/actions/users.py

@@ -58,7 +58,7 @@ def get_users(
             )
             for user in user_data["users"]
         ]
-        if not exclude_primary:
+        if not exclude_primary and "username" in user_data.keys():
             users.append(
                 UserDataUser(
                     username=user_data["username"],
@@ -107,6 +107,12 @@ class PasswordIsEmpty(Exception):
     pass
 
 
+class InvalidConfiguration(Exception):
+    """The userdata is broken"""
+
+    pass
+
+
 def create_user(username: str, password: str):
     if password == "":
         raise PasswordIsEmpty("Password is empty")
@@ -124,6 +130,10 @@ def create_user(username: str, password: str):
 
     with ReadUserData() as user_data:
         ensure_ssh_and_users_fields_exist(user_data)
+        if "username" not in user_data.keys():
+            raise InvalidConfiguration(
+                "Broken config: Admin name is not defined. Consider recovery or add it manually"
+            )
         if username == user_data["username"]:
             raise UserAlreadyExists("User already exists")
         if username in [user["username"] for user in user_data["users"]]:

selfprivacy_api/app.py

@@ -9,14 +9,7 @@ import uvicorn
 from selfprivacy_api.dependencies import get_api_version
 from selfprivacy_api.graphql.schema import schema
 from selfprivacy_api.migrations import run_migrations
-from selfprivacy_api.restic_controller.tasks import init_restic
 
-from selfprivacy_api.rest import (
-    system,
-    users,
-    api_auth,
-    services,
-)
 
 app = FastAPI()
 
@@ -33,10 +26,6 @@ app.add_middleware(
 )
 
 
-app.include_router(system.router)
-app.include_router(users.router)
-app.include_router(api_auth.router)
-app.include_router(services.router)
 app.include_router(graphql_app, prefix="/graphql")
 
 
@@ -49,7 +38,6 @@ async def get_version():
 @app.on_event("startup")
 async def startup():
     run_migrations()
-    init_restic()
 
 
 if __name__ == "__main__":

selfprivacy_api/backup/__init__.py

@@ -0,0 +1,741 @@
+"""
+This module contains the controller class for backups.
+"""
+from datetime import datetime, timedelta, timezone
+import time
+import os
+from os import statvfs
+from typing import Callable, List, Optional
+
+from selfprivacy_api.services import (
+    get_service_by_id,
+    get_all_services,
+)
+from selfprivacy_api.services.service import (
+    Service,
+    ServiceStatus,
+    StoppedService,
+)
+
+from selfprivacy_api.jobs import Jobs, JobStatus, Job
+
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+from selfprivacy_api.graphql.common_types.backup import (
+    RestoreStrategy,
+    BackupReason,
+    AutobackupQuotas,
+)
+
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+
+from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
+from selfprivacy_api.backup.providers import get_provider
+from selfprivacy_api.backup.storage import Storage
+from selfprivacy_api.backup.jobs import (
+    get_backup_job,
+    get_backup_fail,
+    add_backup_job,
+    get_restore_job,
+    add_restore_job,
+)
+
+
+BACKUP_PROVIDER_ENVS = {
+    "kind": "BACKUP_KIND",
+    "login": "BACKUP_LOGIN",
+    "key": "BACKUP_KEY",
+    "location": "BACKUP_LOCATION",
+}
+
+AUTOBACKUP_JOB_EXPIRATION_SECONDS = 60 * 60  # one hour
+
+
+class NotDeadError(AssertionError):
+    """
+    This error is raised when we try to back up a service that is not dead yet.
+    """
+
+    def __init__(self, service: Service):
+        self.service_name = service.get_id()
+        super().__init__()
+
+    def __str__(self):
+        return f"""
+        Service {self.service_name} should be either stopped or dead from
+        an error before we back up.
+        Normally, this error is unreachable because we do try ensure this.
+        Apparently, not this time.
+        """
+
+
+class RotationBucket:
+    """
+    Bucket object used for rotation.
+    Has the following mutable fields:
+    - the counter, int
+    - the lambda function which takes datetime and the int and returns the int
+    - the last, int
+    """
+
+    def __init__(self, counter: int, last: int, rotation_lambda):
+        self.counter: int = counter
+        self.last: int = last
+        self.rotation_lambda: Callable[[datetime, int], int] = rotation_lambda
+
+    def __str__(self) -> str:
+        return f"Bucket(counter={self.counter}, last={self.last})"
+
+
+class Backups:
+    """A stateless controller class for backups"""
+
+    # Providers
+
+    @staticmethod
+    def provider() -> AbstractBackupProvider:
+        """
+        Returns the current backup storage provider.
+        """
+        return Backups._lookup_provider()
+
+    @staticmethod
+    def set_provider(
+        kind: BackupProviderEnum,
+        login: str,
+        key: str,
+        location: str,
+        repo_id: str = "",
+    ) -> None:
+        """
+        Sets the new configuration of the backup storage provider.
+
+        In case of `BackupProviderEnum.BACKBLAZE`, the `login` is the key ID,
+        the `key` is the key itself, and the `location` is the bucket name and
+        the `repo_id` is the bucket ID.
+        """
+        provider: AbstractBackupProvider = Backups._construct_provider(
+            kind,
+            login,
+            key,
+            location,
+            repo_id,
+        )
+        Storage.store_provider(provider)
+
+    @staticmethod
+    def reset() -> None:
+        """
+        Deletes all the data about the backup storage provider.
+        """
+        Storage.reset()
+
+    @staticmethod
+    def _lookup_provider() -> AbstractBackupProvider:
+        redis_provider = Backups._load_provider_redis()
+        if redis_provider is not None:
+            return redis_provider
+
+        none_provider = Backups._construct_provider(
+            BackupProviderEnum.NONE, login="", key="", location=""
+        )
+        Storage.store_provider(none_provider)
+        return none_provider
+
+    @staticmethod
+    def set_provider_from_envs():
+        for env in BACKUP_PROVIDER_ENVS.values():
+            if env not in os.environ.keys():
+                raise ValueError(
+                    f"Cannot set backup provider from envs, there is no {env} set"
+                )
+
+        kind_str = os.environ[BACKUP_PROVIDER_ENVS["kind"]]
+        kind_enum = BackupProviderEnum[kind_str]
+        provider = Backups._construct_provider(
+            kind=kind_enum,
+            login=os.environ[BACKUP_PROVIDER_ENVS["login"]],
+            key=os.environ[BACKUP_PROVIDER_ENVS["key"]],
+            location=os.environ[BACKUP_PROVIDER_ENVS["location"]],
+        )
+        Storage.store_provider(provider)
+
+    @staticmethod
+    def _construct_provider(
+        kind: BackupProviderEnum,
+        login: str,
+        key: str,
+        location: str,
+        repo_id: str = "",
+    ) -> AbstractBackupProvider:
+        provider_class = get_provider(kind)
+
+        return provider_class(
+            login=login,
+            key=key,
+            location=location,
+            repo_id=repo_id,
+        )
+
+    @staticmethod
+    def _load_provider_redis() -> Optional[AbstractBackupProvider]:
+        provider_model = Storage.load_provider()
+        if provider_model is None:
+            return None
+        return Backups._construct_provider(
+            BackupProviderEnum[provider_model.kind],
+            provider_model.login,
+            provider_model.key,
+            provider_model.location,
+            provider_model.repo_id,
+        )
+
+    # Init
+
+    @staticmethod
+    def init_repo() -> None:
+        """
+        Initializes the backup repository. This is required once per repo.
+        """
+        Backups.provider().backupper.init()
+        Storage.mark_as_init()
+
+    @staticmethod
+    def erase_repo() -> None:
+        """
+        Completely empties the remote
+        """
+        Backups.provider().backupper.erase_repo()
+        Storage.mark_as_uninitted()
+
+    @staticmethod
+    def is_initted() -> bool:
+        """
+        Returns whether the backup repository is initialized or not.
+        If it is not initialized, we cannot back up and probably should
+        call `init_repo` first.
+        """
+        if Storage.has_init_mark():
+            return True
+
+        initted = Backups.provider().backupper.is_initted()
+        if initted:
+            Storage.mark_as_init()
+            return True
+
+        return False
+
+    # Backup
+
+    @staticmethod
+    def back_up(
+        service: Service, reason: BackupReason = BackupReason.EXPLICIT
+    ) -> Snapshot:
+        """The top-level function to back up a service
+        If it fails for any reason at all, it should both mark job as
+        errored and re-raise an error"""
+
+        job = get_backup_job(service)
+        if job is None:
+            job = add_backup_job(service)
+        Jobs.update(job, status=JobStatus.RUNNING)
+
+        try:
+            if service.can_be_backed_up() is False:
+                raise ValueError("cannot backup a non-backuppable service")
+            folders = service.get_folders()
+            service_name = service.get_id()
+            service.pre_backup()
+            snapshot = Backups.provider().backupper.start_backup(
+                folders,
+                service_name,
+                reason=reason,
+            )
+
+            Backups._on_new_snapshot_created(service_name, snapshot)
+            if reason == BackupReason.AUTO:
+                Backups._prune_auto_snaps(service)
+            service.post_restore()
+        except Exception as error:
+            Jobs.update(job, status=JobStatus.ERROR, error=str(error))
+            raise error
+
+        Jobs.update(job, status=JobStatus.FINISHED)
+        if reason in [BackupReason.AUTO, BackupReason.PRE_RESTORE]:
+            Jobs.set_expiration(job, AUTOBACKUP_JOB_EXPIRATION_SECONDS)
+        return Backups.sync_date_from_cache(snapshot)
+
+    @staticmethod
+    def sync_date_from_cache(snapshot: Snapshot) -> Snapshot:
+        """
+        Our snapshot creation dates are different from those on server by a tiny amount.
+        This is a convenience, maybe it is better to write a special comparison
+        function for snapshots
+        """
+        return Storage.get_cached_snapshot_by_id(snapshot.id)
+
+    @staticmethod
+    def _auto_snaps(service):
+        return [
+            snap
+            for snap in Backups.get_snapshots(service)
+            if snap.reason == BackupReason.AUTO
+        ]
+
+    @staticmethod
+    def _prune_snaps_with_quotas(snapshots: List[Snapshot]) -> List[Snapshot]:
+        # Function broken out for testability
+        # Sorting newest first
+        sorted_snaps = sorted(snapshots, key=lambda s: s.created_at, reverse=True)
+        quotas: AutobackupQuotas = Backups.autobackup_quotas()
+
+        buckets: list[RotationBucket] = [
+            RotationBucket(
+                quotas.last,  # type: ignore
+                -1,
+                lambda _, index: index,
+            ),
+            RotationBucket(
+                quotas.daily,  # type: ignore
+                -1,
+                lambda date, _: date.year * 10000 + date.month * 100 + date.day,
+            ),
+            RotationBucket(
+                quotas.weekly,  # type: ignore
+                -1,
+                lambda date, _: date.year * 100 + date.isocalendar()[1],
+            ),
+            RotationBucket(
+                quotas.monthly,  # type: ignore
+                -1,
+                lambda date, _: date.year * 100 + date.month,
+            ),
+            RotationBucket(
+                quotas.yearly,  # type: ignore
+                -1,
+                lambda date, _: date.year,
+            ),
+        ]
+
+        new_snaplist: List[Snapshot] = []
+        for i, snap in enumerate(sorted_snaps):
+            keep_snap = False
+            for bucket in buckets:
+                if (bucket.counter > 0) or (bucket.counter == -1):
+                    val = bucket.rotation_lambda(snap.created_at, i)
+                    if (val != bucket.last) or (i == len(sorted_snaps) - 1):
+                        bucket.last = val
+                        if bucket.counter > 0:
+                            bucket.counter -= 1
+                        if not keep_snap:
+                            new_snaplist.append(snap)
+                        keep_snap = True
+
+        return new_snaplist
+
+    @staticmethod
+    def _prune_auto_snaps(service) -> None:
+        # Not very testable by itself, so most testing is going on Backups._prune_snaps_with_quotas
+        # We can still test total limits and, say, daily limits
+
+        auto_snaps = Backups._auto_snaps(service)
+        new_snaplist = Backups._prune_snaps_with_quotas(auto_snaps)
+
+        deletable_snaps = [snap for snap in auto_snaps if snap not in new_snaplist]
+        Backups.forget_snapshots(deletable_snaps)
+
+    @staticmethod
+    def _standardize_quotas(i: int) -> int:
+        if i <= -1:
+            i = -1
+        return i
+
+    @staticmethod
+    def autobackup_quotas() -> AutobackupQuotas:
+        """0 means do not keep, -1 means unlimited"""
+
+        return Storage.autobackup_quotas()
+
+    @staticmethod
+    def set_autobackup_quotas(quotas: AutobackupQuotas) -> None:
+        """0 means do not keep, -1 means unlimited"""
+
+        Storage.set_autobackup_quotas(
+            AutobackupQuotas(
+                last=Backups._standardize_quotas(quotas.last),  # type: ignore
+                daily=Backups._standardize_quotas(quotas.daily),  # type: ignore
+                weekly=Backups._standardize_quotas(quotas.weekly),  # type: ignore
+                monthly=Backups._standardize_quotas(quotas.monthly),  # type: ignore
+                yearly=Backups._standardize_quotas(quotas.yearly),  # type: ignore
+            )
+        )
+        # do not prune all autosnaps right away, this will be done by an async task
+
+    @staticmethod
+    def prune_all_autosnaps() -> None:
+        for service in get_all_services():
+            Backups._prune_auto_snaps(service)
+
+    # Restoring
+
+    @staticmethod
+    def _ensure_queued_restore_job(service, snapshot) -> Job:
+        job = get_restore_job(service)
+        if job is None:
+            job = add_restore_job(snapshot)
+
+        Jobs.update(job, status=JobStatus.CREATED)
+        return job
+
+    @staticmethod
+    def _inplace_restore(
+        service: Service,
+        snapshot: Snapshot,
+        job: Job,
+    ) -> None:
+        Jobs.update(
+            job, status=JobStatus.CREATED, status_text="Waiting for pre-restore backup"
+        )
+        failsafe_snapshot = Backups.back_up(service, BackupReason.PRE_RESTORE)
+
+        Jobs.update(
+            job, status=JobStatus.RUNNING, status_text=f"Restoring from {snapshot.id}"
+        )
+        try:
+            Backups._restore_service_from_snapshot(
+                service,
+                snapshot.id,
+                verify=False,
+            )
+        except Exception as error:
+            Jobs.update(
+                job,
+                status=JobStatus.ERROR,
+                status_text=f"Restore failed with {str(error)}, reverting to {failsafe_snapshot.id}",
+            )
+            Backups._restore_service_from_snapshot(
+                service, failsafe_snapshot.id, verify=False
+            )
+            Jobs.update(
+                job,
+                status=JobStatus.ERROR,
+                status_text=f"Restore failed with {str(error)}, reverted to {failsafe_snapshot.id}",
+            )
+            raise error
+
+    @staticmethod
+    def restore_snapshot(
+        snapshot: Snapshot, strategy=RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE
+    ) -> None:
+        """Restores a snapshot to its original service using the given strategy"""
+        service = get_service_by_id(snapshot.service_name)
+        if service is None:
+            raise ValueError(
+                f"snapshot has a nonexistent service: {snapshot.service_name}"
+            )
+        job = Backups._ensure_queued_restore_job(service, snapshot)
+
+        try:
+            Backups._assert_restorable(snapshot)
+            Jobs.update(
+                job, status=JobStatus.RUNNING, status_text="Stopping the service"
+            )
+            with StoppedService(service):
+                Backups.assert_dead(service)
+                if strategy == RestoreStrategy.INPLACE:
+                    Backups._inplace_restore(service, snapshot, job)
+                else:  # verify_before_download is our default
+                    Jobs.update(
+                        job,
+                        status=JobStatus.RUNNING,
+                        status_text=f"Restoring from {snapshot.id}",
+                    )
+                    Backups._restore_service_from_snapshot(
+                        service, snapshot.id, verify=True
+                    )
+
+                service.post_restore()
+                Jobs.update(
+                    job,
+                    status=JobStatus.RUNNING,
+                    progress=90,
+                    status_text="Restarting the service",
+                )
+
+        except Exception as error:
+            Jobs.update(job, status=JobStatus.ERROR, status_text=str(error))
+            raise error
+
+        Jobs.update(job, status=JobStatus.FINISHED)
+
+    @staticmethod
+    def _assert_restorable(
+        snapshot: Snapshot, strategy=RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE
+    ) -> None:
+        service = get_service_by_id(snapshot.service_name)
+        if service is None:
+            raise ValueError(
+                f"snapshot has a nonexistent service: {snapshot.service_name}"
+            )
+
+        restored_snap_size = Backups.snapshot_restored_size(snapshot.id)
+
+        if strategy == RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE:
+            needed_space = restored_snap_size
+        elif strategy == RestoreStrategy.INPLACE:
+            needed_space = restored_snap_size - service.get_storage_usage()
+        else:
+            raise NotImplementedError(
+                """
+            We do not know if there is enough space for restoration because
+            there is some novel restore strategy used!
+            This is a developer's fault, open an issue please
+            """
+            )
+        available_space = Backups.space_usable_for_service(service)
+        if needed_space > available_space:
+            raise ValueError(
+                f"we only have {available_space} bytes "
+                f"but snapshot needs {needed_space}"
+            )
+
+    @staticmethod
+    def _restore_service_from_snapshot(
+        service: Service,
+        snapshot_id: str,
+        verify=True,
+    ) -> None:
+        folders = service.get_folders()
+
+        Backups.provider().backupper.restore_from_backup(
+            snapshot_id,
+            folders,
+            verify=verify,
+        )
+
+    # Snapshots
+
+    @staticmethod
+    def get_snapshots(service: Service) -> List[Snapshot]:
+        """Returns all snapshots for a given service"""
+        snapshots = Backups.get_all_snapshots()
+        service_id = service.get_id()
+        return list(
+            filter(
+                lambda snap: snap.service_name == service_id,
+                snapshots,
+            )
+        )
+
+    @staticmethod
+    def get_all_snapshots() -> List[Snapshot]:
+        """Returns all snapshots"""
+        # When we refresh our cache:
+        # 1. Manually
+        # 2. On timer
+        # 3. On new snapshot
+        # 4. On snapshot deletion
+
+        return Storage.get_cached_snapshots()
+
+    @staticmethod
+    def get_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]:
+        """Returns a backup snapshot by its id"""
+        snap = Storage.get_cached_snapshot_by_id(snapshot_id)
+        if snap is not None:
+            return snap
+
+        # Possibly our cache entry got invalidated, let's try one more time
+        Backups.force_snapshot_cache_reload()
+        snap = Storage.get_cached_snapshot_by_id(snapshot_id)
+
+        return snap
+
+    @staticmethod
+    def forget_snapshots(snapshots: List[Snapshot]) -> None:
+        """
+        Deletes a batch of snapshots from the repo and syncs cache
+        Optimized
+        """
+        ids = [snapshot.id for snapshot in snapshots]
+        Backups.provider().backupper.forget_snapshots(ids)
+
+        Backups.force_snapshot_cache_reload()
+
+    @staticmethod
+    def forget_snapshot(snapshot: Snapshot) -> None:
+        """Deletes a snapshot from the repo and from cache"""
+        Backups.forget_snapshots([snapshot])
+
+    @staticmethod
+    def forget_all_snapshots():
+        """
+        Mark all snapshots we have made for deletion and make them inaccessible
+        (this is done by cloud, we only issue a command)
+        """
+        Backups.forget_snapshots(Backups.get_all_snapshots())
+
+    @staticmethod
+    def force_snapshot_cache_reload() -> None:
+        """
+        Forces a reload of the snapshot cache.
+
+        This may be an expensive operation, so use it wisely.
+        User pays for the API calls.
+        """
+        upstream_snapshots = Backups.provider().backupper.get_snapshots()
+        Storage.invalidate_snapshot_storage()
+        for snapshot in upstream_snapshots:
+            Storage.cache_snapshot(snapshot)
+
+    @staticmethod
+    def snapshot_restored_size(snapshot_id: str) -> int:
+        """Returns the size of the snapshot"""
+        return Backups.provider().backupper.restored_size(
+            snapshot_id,
+        )
+
+    @staticmethod
+    def _on_new_snapshot_created(service_id: str, snapshot: Snapshot) -> None:
+        """What do we do with a snapshot that is just made?"""
+        # non-expiring timestamp of the last
+        Storage.store_last_timestamp(service_id, snapshot)
+        Backups.force_snapshot_cache_reload()
+
+    # Autobackup
+
+    @staticmethod
+    def autobackup_period_minutes() -> Optional[int]:
+        """None means autobackup is disabled"""
+        return Storage.autobackup_period_minutes()
+
+    @staticmethod
+    def set_autobackup_period_minutes(minutes: int) -> None:
+        """
+        0 and negative numbers are equivalent to disable.
+        Setting to a positive number may result in a backup very soon
+        if some services are not backed up.
+        """
+        if minutes <= 0:
+            Backups.disable_all_autobackup()
+            return
+        Storage.store_autobackup_period_minutes(minutes)
+
+    @staticmethod
+    def disable_all_autobackup() -> None:
+        """
+        Disables all automatic backing up,
+        but does not change per-service settings
+        """
+        Storage.delete_backup_period()
+
+    @staticmethod
+    def is_time_to_backup(time: datetime) -> bool:
+        """
+        Intended as a time validator for huey cron scheduler
+        of automatic backups
+        """
+
+        return Backups.services_to_back_up(time) != []
+
+    @staticmethod
+    def services_to_back_up(time: datetime) -> List[Service]:
+        """Returns a list of services that should be backed up at a given time"""
+        return [
+            service
+            for service in get_all_services()
+            if Backups.is_time_to_backup_service(service, time)
+        ]
+
+    @staticmethod
+    def get_last_backed_up(service: Service) -> Optional[datetime]:
+        """Get a timezone-aware time of the last backup of a service"""
+        return Storage.get_last_backup_time(service.get_id())
+
+    @staticmethod
+    def get_last_backup_error_time(service: Service) -> Optional[datetime]:
+        """Get a timezone-aware time of the last backup of a service"""
+        job = get_backup_fail(service)
+        if job is not None:
+            datetime_created = job.created_at
+            if datetime_created.tzinfo is None:
+                # assume it is in localtime
+                offset = timedelta(seconds=time.localtime().tm_gmtoff)
+                datetime_created = datetime_created - offset
+                return datetime.combine(
+                    datetime_created.date(), datetime_created.time(), timezone.utc
+                )
+            return datetime_created
+        return None
+
+    @staticmethod
+    def is_time_to_backup_service(service: Service, time: datetime):
+        """Returns True if it is time to back up a service"""
+        period = Backups.autobackup_period_minutes()
+        if period is None:
+            return False
+
+        if not service.is_enabled():
+            return False
+        if not service.can_be_backed_up():
+            return False
+
+        last_error = Backups.get_last_backup_error_time(service)
+
+        if last_error is not None:
+            if time < last_error + timedelta(seconds=AUTOBACKUP_JOB_EXPIRATION_SECONDS):
+                return False
+
+        last_backup = Backups.get_last_backed_up(service)
+
+        # Queue a backup immediately if there are no previous backups
+        if last_backup is None:
+            return True
+
+        if time > last_backup + timedelta(minutes=period):
+            return True
+
+        return False
+
+    # Helpers
+
+    @staticmethod
+    def space_usable_for_service(service: Service) -> int:
+        """
+        Returns the amount of space available on the volume the given
+        service is located on.
+        """
+        folders = service.get_folders()
+        if folders == []:
+            raise ValueError("unallocated service", service.get_id())
+
+        # We assume all folders of one service live at the same volume
+        fs_info = statvfs(folders[0])
+        usable_bytes = fs_info.f_frsize * fs_info.f_bavail
+        return usable_bytes
+
+    @staticmethod
+    def set_localfile_repo(file_path: str):
+        """Used by tests to set a local folder as a backup repo"""
+        # pylint: disable-next=invalid-name
+        ProviderClass = get_provider(BackupProviderEnum.FILE)
+        provider = ProviderClass(
+            login="",
+            key="",
+            location=file_path,
+            repo_id="",
+        )
+        Storage.store_provider(provider)
+
+    @staticmethod
+    def assert_dead(service: Service):
+        """
+        Checks if a service is dead and can be safely restored from a snapshot.
+        """
+        if service.get_status() not in [
+            ServiceStatus.INACTIVE,
+            ServiceStatus.FAILED,
+        ]:
+            raise NotDeadError(service)

selfprivacy_api/backup/backuppers/__init__.py

@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.graphql.common_types.backup import BackupReason
+
+
+class AbstractBackupper(ABC):
+    """Abstract class for backuppers"""
+
+    # flake8: noqa: B027
+    def __init__(self) -> None:
+        pass
+
+    @abstractmethod
+    def is_initted(self) -> bool:
+        """Returns true if the repository is initted"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def set_creds(self, account: str, key: str, repo: str) -> None:
+        """Set the credentials for the backupper"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def start_backup(
+        self,
+        folders: List[str],
+        service_name: str,
+        reason: BackupReason = BackupReason.EXPLICIT,
+    ) -> Snapshot:
+        """Start a backup of the given folders"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_snapshots(self) -> List[Snapshot]:
+        """Get all snapshots from the repo"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def init(self) -> None:
+        """Initialize the repository"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def erase_repo(self) -> None:
+        """Completely empties the remote"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def restore_from_backup(
+        self,
+        snapshot_id: str,
+        folders: List[str],
+        verify=True,
+    ) -> None:
+        """Restore a target folder using a snapshot"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def restored_size(self, snapshot_id: str) -> int:
+        """Get the size of the restored snapshot"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def forget_snapshot(self, snapshot_id) -> None:
+        """Forget a snapshot"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def forget_snapshots(self, snapshot_ids: List[str]) -> None:
+        """Maybe optimized deletion of a batch of snapshots, just cycling if unsupported"""
+        raise NotImplementedError

selfprivacy_api/backup/backuppers/none_backupper.py

@@ -0,0 +1,45 @@
+from typing import List
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.backup.backuppers import AbstractBackupper
+from selfprivacy_api.graphql.common_types.backup import BackupReason
+
+
+class NoneBackupper(AbstractBackupper):
+    """A backupper that does nothing"""
+
+    def is_initted(self, repo_name: str = "") -> bool:
+        return False
+
+    def set_creds(self, account: str, key: str, repo: str):
+        pass
+
+    def start_backup(
+        self, folders: List[str], tag: str, reason: BackupReason = BackupReason.EXPLICIT
+    ):
+        raise NotImplementedError
+
+    def get_snapshots(self) -> List[Snapshot]:
+        """Get all snapshots from the repo"""
+        return []
+
+    def init(self):
+        raise NotImplementedError
+
+    def erase_repo(self) -> None:
+        """Completely empties the remote"""
+        # this one is already empty
+        pass
+
+    def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True):
+        """Restore a target folder using a snapshot"""
+        raise NotImplementedError
+
+    def restored_size(self, snapshot_id: str) -> int:
+        raise NotImplementedError
+
+    def forget_snapshot(self, snapshot_id):
+        raise NotImplementedError("forget_snapshot")
+
+    def forget_snapshots(self, snapshots):
+        raise NotImplementedError("forget_snapshots")

selfprivacy_api/backup/backuppers/restic_backupper.py

@@ -0,0 +1,554 @@
+from __future__ import annotations
+
+import subprocess
+import json
+import datetime
+import tempfile
+
+from typing import List, Optional, TypeVar, Callable
+from collections.abc import Iterable
+from json.decoder import JSONDecodeError
+from os.path import exists, join
+from os import mkdir
+from shutil import rmtree
+
+from selfprivacy_api.graphql.common_types.backup import BackupReason
+from selfprivacy_api.backup.util import output_yielder, sync
+from selfprivacy_api.backup.backuppers import AbstractBackupper
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.backup.jobs import get_backup_job
+from selfprivacy_api.services import get_service_by_id
+from selfprivacy_api.jobs import Jobs, JobStatus, Job
+
+from selfprivacy_api.backup.local_secret import LocalBackupSecret
+
+SHORT_ID_LEN = 8
+
+T = TypeVar("T", bound=Callable)
+
+
+def unlocked_repo(func: T) -> T:
+    """unlock repo and retry if it appears to be locked"""
+
+    def inner(self: ResticBackupper, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except Exception as error:
+            if "unable to create lock" in str(error):
+                self.unlock()
+                return func(self, *args, **kwargs)
+            else:
+                raise error
+
+    # Above, we manually guarantee that the type returned is compatible.
+    return inner  # type: ignore
+
+
+class ResticBackupper(AbstractBackupper):
+    def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None:
+        self.login_flag = login_flag
+        self.key_flag = key_flag
+        self.storage_type = storage_type
+        self.account = ""
+        self.key = ""
+        self.repo = ""
+        super().__init__()
+
+    def set_creds(self, account: str, key: str, repo: str) -> None:
+        self.account = account
+        self.key = key
+        self.repo = repo
+
+    def restic_repo(self) -> str:
+        # https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone
+        # https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5
+        return f"rclone:{self.rclone_repo()}"
+
+    def rclone_repo(self) -> str:
+        return f"{self.storage_type}{self.repo}"
+
+    def rclone_args(self):
+        return "rclone.args=serve restic --stdio " + " ".join(
+            self.backend_rclone_args()
+        )
+
+    def backend_rclone_args(self) -> list[str]:
+        args = []
+        if self.account != "":
+            acc_args = [self.login_flag, self.account]
+            args.extend(acc_args)
+        if self.key != "":
+            key_args = [self.key_flag, self.key]
+            args.extend(key_args)
+        return args
+
+    def _password_command(self):
+        return f"echo {LocalBackupSecret.get()}"
+
+    def restic_command(self, *args, tags: Optional[List[str]] = None) -> List[str]:
+        """
+        Construct a restic command against the currently configured repo
+        Can support [nested] arrays as arguments, will flatten them into the final commmand
+        """
+        if tags is None:
+            tags = []
+
+        command = [
+            "restic",
+            "-o",
+            self.rclone_args(),
+            "-r",
+            self.restic_repo(),
+            "--password-command",
+            self._password_command(),
+        ]
+        if tags != []:
+            for tag in tags:
+                command.extend(
+                    [
+                        "--tag",
+                        tag,
+                    ]
+                )
+        if args:
+            command.extend(ResticBackupper.__flatten_list(args))
+        return command
+
+    def erase_repo(self) -> None:
+        """Fully erases repo on remote, can be reinitted again"""
+        command = [
+            "rclone",
+            "purge",
+            self.rclone_repo(),
+        ]
+        backend_args = self.backend_rclone_args()
+        if backend_args:
+            command.extend(backend_args)
+
+        with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle:
+            output = handle.communicate()[0].decode("utf-8")
+            if handle.returncode != 0:
+                raise ValueError(
+                    "purge exited with errorcode",
+                    handle.returncode,
+                    ":",
+                    output,
+                )
+
+    @staticmethod
+    def __flatten_list(list_to_flatten):
+        """string-aware list flattener"""
+        result = []
+        for item in list_to_flatten:
+            if isinstance(item, Iterable) and not isinstance(item, str):
+                result.extend(ResticBackupper.__flatten_list(item))
+                continue
+            result.append(item)
+        return result
+
+    @staticmethod
+    def _run_backup_command(
+        backup_command: List[str], job: Optional[Job]
+    ) -> List[dict]:
+        """And handle backup output"""
+        messages = []
+        output = []
+        restic_reported_error = False
+
+        for raw_message in output_yielder(backup_command):
+            if "ERROR:" in raw_message:
+                restic_reported_error = True
+            output.append(raw_message)
+
+            if not restic_reported_error:
+                message = ResticBackupper.parse_message(raw_message, job)
+                messages.append(message)
+
+        if restic_reported_error:
+            raise ValueError(
+                "Restic returned error(s): ",
+                output,
+            )
+
+        return messages
+
+    @staticmethod
+    def _replace_in_array(array: List[str], target, replacement) -> None:
+        if target == "":
+            return
+
+        for i, value in enumerate(array):
+            if target in value:
+                array[i] = array[i].replace(target, replacement)
+
+    def _censor_command(self, command: List[str]) -> List[str]:
+        result = command.copy()
+        ResticBackupper._replace_in_array(result, self.key, "CENSORED")
+        ResticBackupper._replace_in_array(result, LocalBackupSecret.get(), "CENSORED")
+        return result
+
+    @staticmethod
+    def _get_backup_job(service_name: str) -> Optional[Job]:
+        service = get_service_by_id(service_name)
+        if service is None:
+            raise ValueError("No service with id ", service_name)
+
+        return get_backup_job(service)
+
+    @unlocked_repo
+    def start_backup(
+        self,
+        folders: List[str],
+        service_name: str,
+        reason: BackupReason = BackupReason.EXPLICIT,
+    ) -> Snapshot:
+        """
+        Start backup with restic
+        """
+        assert len(folders) != 0
+
+        job = ResticBackupper._get_backup_job(service_name)
+
+        tags = [service_name, reason.value]
+        backup_command = self.restic_command(
+            "backup",
+            "--json",
+            folders,
+            tags=tags,
+        )
+
+        try:
+            messages = ResticBackupper._run_backup_command(backup_command, job)
+
+            id = ResticBackupper._snapshot_id_from_backup_messages(messages)
+            return Snapshot(
+                created_at=datetime.datetime.now(datetime.timezone.utc),
+                id=id,
+                service_name=service_name,
+                reason=reason,
+            )
+
+        except ValueError as error:
+            raise ValueError(
+                "Could not create a snapshot: ",
+                str(error),
+                "command: ",
+                self._censor_command(backup_command),
+            ) from error
+
+    @staticmethod
+    def _snapshot_id_from_backup_messages(messages) -> str:
+        for message in messages:
+            if message["message_type"] == "summary":
+                # There is a discrepancy between versions of restic/rclone
+                # Some report short_id in this field and some full
+                return message["snapshot_id"][0:SHORT_ID_LEN]
+
+        raise ValueError("no summary message in restic json output")
+
+    @staticmethod
+    def parse_message(raw_message_line: str, job: Optional[Job] = None) -> dict:
+        message = ResticBackupper.parse_json_output(raw_message_line)
+        if not isinstance(message, dict):
+            raise ValueError("we have too many messages on one line?")
+        if message["message_type"] == "status":
+            if job is not None:  # only update status if we run under some job
+                Jobs.update(
+                    job,
+                    JobStatus.RUNNING,
+                    progress=int(message["percent_done"] * 100),
+                )
+        return message
+
+    def init(self) -> None:
+        init_command = self.restic_command(
+            "init",
+        )
+        with subprocess.Popen(
+            init_command,
+            shell=False,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        ) as process_handle:
+            output = process_handle.communicate()[0].decode("utf-8")
+            if "created restic repository" not in output:
+                raise ValueError("cannot init a repo: " + output)
+
+    @unlocked_repo
+    def is_initted(self) -> bool:
+        command = self.restic_command(
+            "check",
+        )
+
+        with subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            shell=False,
+            stderr=subprocess.STDOUT,
+        ) as handle:
+            output = handle.communicate()[0].decode("utf-8")
+            if handle.returncode != 0:
+                if "unable to create lock" in output:
+                    raise ValueError("Stale lock detected: ", output)
+                return False
+            return True
+
+    def unlock(self) -> None:
+        """Remove stale locks."""
+        command = self.restic_command(
+            "unlock",
+        )
+
+        with subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            shell=False,
+            stderr=subprocess.STDOUT,
+        ) as handle:
+            # communication forces to complete and for returncode to get defined
+            output = handle.communicate()[0].decode("utf-8")
+            if handle.returncode != 0:
+                raise ValueError("cannot unlock the backup repository: ", output)
+
+    def lock(self) -> None:
+        """
+        Introduce a stale lock.
+        Mainly for testing purposes.
+        Double lock is supposed to fail
+        """
+        command = self.restic_command(
+            "check",
+        )
+
+        # using temporary cache in /run/user/1000/restic-check-cache-817079729
+        # repository 9639c714 opened (repository version 2) successfully, password is correct
+        # created new cache in /run/user/1000/restic-check-cache-817079729
+        # create exclusive lock for repository
+        # load indexes
+        # check all packs
+        # check snapshots, trees and blobs
+        # [0:00] 100.00%  1 / 1 snapshots
+        # no errors were found
+
+        try:
+            for line in output_yielder(command):
+                if "indexes" in line:
+                    break
+                if "unable" in line:
+                    raise ValueError(line)
+        except Exception as error:
+            raise ValueError("could not lock repository") from error
+
+    @unlocked_repo
+    def restored_size(self, snapshot_id: str) -> int:
+        """
+        Size of a snapshot
+        """
+        command = self.restic_command(
+            "stats",
+            snapshot_id,
+            "--json",
+        )
+
+        with subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            shell=False,
+        ) as handle:
+            output = handle.communicate()[0].decode("utf-8")
+            try:
+                parsed_output = ResticBackupper.parse_json_output(output)
+                return parsed_output["total_size"]
+            except ValueError as error:
+                raise ValueError("cannot restore a snapshot: " + output) from error
+
+    @unlocked_repo
+    def restore_from_backup(
+        self,
+        snapshot_id,
+        folders: List[str],
+        verify=True,
+    ) -> None:
+        """
+        Restore from backup with restic
+        """
+        if folders is None or folders == []:
+            raise ValueError("cannot restore without knowing where to!")
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verify:
+                self._raw_verified_restore(snapshot_id, target=temp_dir)
+                snapshot_root = temp_dir
+                for folder in folders:
+                    src = join(snapshot_root, folder.strip("/"))
+                    if not exists(src):
+                        raise ValueError(
+                            f"No such path: {src}. We tried to find {folder}"
+                        )
+                    dst = folder
+                    sync(src, dst)
+
+            else:  # attempting inplace restore
+                for folder in folders:
+                    rmtree(folder)
+                    mkdir(folder)
+                self._raw_verified_restore(snapshot_id, target="/")
+                return
+
+    def _raw_verified_restore(self, snapshot_id, target="/"):
+        """barebones restic restore"""
+        restore_command = self.restic_command(
+            "restore", snapshot_id, "--target", target, "--verify"
+        )
+
+        with subprocess.Popen(
+            restore_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            shell=False,
+        ) as handle:
+            # for some reason restore does not support
+            # nice reporting of progress via json
+            output = handle.communicate()[0].decode("utf-8")
+            if "restoring" not in output:
+                raise ValueError("cannot restore a snapshot: " + output)
+
+            assert (
+                handle.returncode is not None
+            )  # none should be impossible after communicate
+            if handle.returncode != 0:
+                raise ValueError(
+                    "restore exited with errorcode",
+                    handle.returncode,
+                    ":",
+                    output,
+                )
+
+    def forget_snapshot(self, snapshot_id: str) -> None:
+        self.forget_snapshots([snapshot_id])
+
+    @unlocked_repo
+    def forget_snapshots(self, snapshot_ids: List[str]) -> None:
+        # in case the backupper program supports batching, otherwise implement it by cycling
+        forget_command = self.restic_command(
+            "forget",
+            [snapshot_ids],
+            # TODO: prune should be done in a separate process
+            "--prune",
+        )
+
+        with subprocess.Popen(
+            forget_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=False,
+        ) as handle:
+            # for some reason restore does not support
+            # nice reporting of progress via json
+            output, err = [
+                string.decode(
+                    "utf-8",
+                )
+                for string in handle.communicate()
+            ]
+
+            if "no matching ID found" in err:
+                raise ValueError(
+                    "trying to delete, but no such snapshot(s): ", snapshot_ids
+                )
+
+            assert (
+                handle.returncode is not None
+            )  # none should be impossible after communicate
+            if handle.returncode != 0:
+                raise ValueError(
+                    "forget exited with errorcode", handle.returncode, ":", output, err
+                )
+
+    def _load_snapshots(self) -> object:
+        """
+        Load list of snapshots from repository
+        raises Value Error if repo does not exist
+        """
+        listing_command = self.restic_command(
+            "snapshots",
+            "--json",
+        )
+
+        with subprocess.Popen(
+            listing_command,
+            shell=False,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        ) as backup_listing_process_descriptor:
+            output = backup_listing_process_descriptor.communicate()[0].decode("utf-8")
+
+        if "Is there a repository at the following location?" in output:
+            raise ValueError("No repository! : " + output)
+        try:
+            return ResticBackupper.parse_json_output(output)
+        except ValueError as error:
+            raise ValueError("Cannot load snapshots: ", output) from error
+
+    @unlocked_repo
+    def get_snapshots(self) -> List[Snapshot]:
+        """Get all snapshots from the repo"""
+        snapshots = []
+
+        for restic_snapshot in self._load_snapshots():
+            # Compatibility with previous snaps:
+            if len(restic_snapshot["tags"]) == 1:
+                reason = BackupReason.EXPLICIT
+            else:
+                reason = restic_snapshot["tags"][1]
+
+            snapshot = Snapshot(
+                id=restic_snapshot["short_id"],
+                created_at=restic_snapshot["time"],
+                service_name=restic_snapshot["tags"][0],
+                reason=reason,
+            )
+
+            snapshots.append(snapshot)
+        return snapshots
+
+    @staticmethod
+    def parse_json_output(output: str) -> object:
+        starting_index = ResticBackupper.json_start(output)
+
+        if starting_index == -1:
+            raise ValueError("There is no json in the restic output: " + output)
+
+        truncated_output = output[starting_index:]
+        json_messages = truncated_output.splitlines()
+        if len(json_messages) == 1:
+            try:
+                return json.loads(truncated_output)
+            except JSONDecodeError as error:
+                raise ValueError(
+                    "There is no json in the restic output : " + output
+                ) from error
+
+        result_array = []
+        for message in json_messages:
+            result_array.append(json.loads(message))
+        return result_array
+
+    @staticmethod
+    def json_start(output: str) -> int:
+        indices = [
+            output.find("["),
+            output.find("{"),
+        ]
+        indices = [x for x in indices if x != -1]
+
+        if indices == []:
+            return -1
+        return min(indices)
+
+    @staticmethod
+    def has_json(output: str) -> bool:
+        if ResticBackupper.json_start(output) == -1:
+            return False
+        return True

selfprivacy_api/backup/jobs.py

@@ -0,0 +1,115 @@
+from typing import Optional, List
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.jobs import Jobs, Job, JobStatus
+from selfprivacy_api.services.service import Service
+from selfprivacy_api.services import get_service_by_id
+
+
+def job_type_prefix(service: Service) -> str:
+    return f"services.{service.get_id()}"
+
+
+def backup_job_type(service: Service) -> str:
+    return f"{job_type_prefix(service)}.backup"
+
+
+def autobackup_job_type() -> str:
+    return "backups.autobackup"
+
+
+def restore_job_type(service: Service) -> str:
+    return f"{job_type_prefix(service)}.restore"
+
+
+def get_jobs_by_service(service: Service) -> List[Job]:
+    result = []
+    for job in Jobs.get_jobs():
+        if job.type_id.startswith(job_type_prefix(service)) and job.status in [
+            JobStatus.CREATED,
+            JobStatus.RUNNING,
+        ]:
+            result.append(job)
+    return result
+
+
+def is_something_running_for(service: Service) -> bool:
+    running_jobs = [
+        job for job in get_jobs_by_service(service) if job.status == JobStatus.RUNNING
+    ]
+    return len(running_jobs) != 0
+
+
+def add_autobackup_job(services: List[Service]) -> Job:
+    service_names = [s.get_display_name() for s in services]
+    pretty_service_list: str = ", ".join(service_names)
+    job = Jobs.add(
+        type_id=autobackup_job_type(),
+        name="Automatic backup",
+        description=f"Scheduled backup for services: {pretty_service_list}",
+    )
+    return job
+
+
+def add_backup_job(service: Service) -> Job:
+    if is_something_running_for(service):
+        message = (
+            f"Cannot start a backup of {service.get_id()}, another operation is running: "
+            + get_jobs_by_service(service)[0].type_id
+        )
+        raise ValueError(message)
+    display_name = service.get_display_name()
+    job = Jobs.add(
+        type_id=backup_job_type(service),
+        name=f"Backup {display_name}",
+        description=f"Backing up {display_name}",
+    )
+    return job
+
+
+def add_restore_job(snapshot: Snapshot) -> Job:
+    service = get_service_by_id(snapshot.service_name)
+    if service is None:
+        raise ValueError(f"no such service: {snapshot.service_name}")
+    if is_something_running_for(service):
+        message = (
+            f"Cannot start a restore of {service.get_id()}, another operation is running: "
+            + get_jobs_by_service(service)[0].type_id
+        )
+        raise ValueError(message)
+    display_name = service.get_display_name()
+    job = Jobs.add(
+        type_id=restore_job_type(service),
+        name=f"Restore {display_name}",
+        description=f"restoring {display_name} from {snapshot.id}",
+    )
+    return job
+
+
+def get_job_by_type(type_id: str) -> Optional[Job]:
+    for job in Jobs.get_jobs():
+        if job.type_id == type_id and job.status in [
+            JobStatus.CREATED,
+            JobStatus.RUNNING,
+        ]:
+            return job
+    return None
+
+
+def get_failed_job_by_type(type_id: str) -> Optional[Job]:
+    for job in Jobs.get_jobs():
+        if job.type_id == type_id and job.status == JobStatus.ERROR:
+            return job
+    return None
+
+
+def get_backup_job(service: Service) -> Optional[Job]:
+    return get_job_by_type(backup_job_type(service))
+
+
+def get_backup_fail(service: Service) -> Optional[Job]:
+    return get_failed_job_by_type(backup_job_type(service))
+
+
+def get_restore_job(service: Service) -> Optional[Job]:
+    return get_job_by_type(restore_job_type(service))

selfprivacy_api/backup/local_secret.py

@@ -0,0 +1,45 @@
+"""Handling of local secret used for encrypted backups.
+Separated out for circular dependency reasons
+"""
+
+from __future__ import annotations
+import secrets
+
+from selfprivacy_api.utils.redis_pool import RedisPool
+
+
+REDIS_KEY = "backup:local_secret"
+
+redis = RedisPool().get_connection()
+
+
+class LocalBackupSecret:
+    @staticmethod
+    def get() -> str:
+        """A secret string which backblaze/other clouds do not know.
+        Serves as encryption key.
+        """
+        if not LocalBackupSecret.exists():
+            LocalBackupSecret.reset()
+        return redis.get(REDIS_KEY)  # type: ignore
+
+    @staticmethod
+    def set(secret: str):
+        redis.set(REDIS_KEY, secret)
+
+    @staticmethod
+    def reset():
+        new_secret = LocalBackupSecret._generate()
+        LocalBackupSecret.set(new_secret)
+
+    @staticmethod
+    def _full_reset():
+        redis.delete(REDIS_KEY)
+
+    @staticmethod
+    def exists() -> bool:
+        return redis.exists(REDIS_KEY) == 1
+
+    @staticmethod
+    def _generate() -> str:
+        return secrets.token_urlsafe(256)

selfprivacy_api/backup/providers/__init__.py

@@ -0,0 +1,31 @@
+from typing import Type
+
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
+
+from selfprivacy_api.backup.providers.backblaze import Backblaze
+from selfprivacy_api.backup.providers.memory import InMemoryBackup
+from selfprivacy_api.backup.providers.local_file import LocalFileBackup
+from selfprivacy_api.backup.providers.none import NoBackups
+
+PROVIDER_MAPPING: dict[BackupProviderEnum, Type[AbstractBackupProvider]] = {
+    BackupProviderEnum.BACKBLAZE: Backblaze,
+    BackupProviderEnum.MEMORY: InMemoryBackup,
+    BackupProviderEnum.FILE: LocalFileBackup,
+    BackupProviderEnum.NONE: NoBackups,
+}
+
+
+def get_provider(
+    provider_type: BackupProviderEnum,
+) -> Type[AbstractBackupProvider]:
+    if provider_type not in PROVIDER_MAPPING.keys():
+        raise LookupError("could not look up provider", provider_type)
+    return PROVIDER_MAPPING[provider_type]
+
+
+def get_kind(provider: AbstractBackupProvider) -> str:
+    """Get the kind of the provider in the form of a string"""
+    return provider.name.value

selfprivacy_api/backup/providers/backblaze.py

@@ -0,0 +1,11 @@
+from .provider import AbstractBackupProvider
+from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+
+
+class Backblaze(AbstractBackupProvider):
+    backupper = ResticBackupper("--b2-account", "--b2-key", ":b2:")
+
+    name = BackupProviderEnum.BACKBLAZE

selfprivacy_api/backup/providers/local_file.py

@@ -0,0 +1,11 @@
+from .provider import AbstractBackupProvider
+from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+
+
+class LocalFileBackup(AbstractBackupProvider):
+    backupper = ResticBackupper("", "", ":local:")
+
+    name = BackupProviderEnum.FILE

selfprivacy_api/backup/providers/memory.py

@@ -0,0 +1,11 @@
+from .provider import AbstractBackupProvider
+from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+
+
+class InMemoryBackup(AbstractBackupProvider):
+    backupper = ResticBackupper("", "", ":memory:")
+
+    name = BackupProviderEnum.MEMORY

selfprivacy_api/backup/providers/none.py

@@ -0,0 +1,11 @@
+from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
+from selfprivacy_api.backup.backuppers.none_backupper import NoneBackupper
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+
+
+class NoBackups(AbstractBackupProvider):
+    backupper = NoneBackupper()
+
+    name = BackupProviderEnum.NONE

selfprivacy_api/backup/providers/provider.py

@@ -0,0 +1,25 @@
+"""
+An abstract class for BackBlaze, S3 etc.
+It assumes that while some providers are supported via restic/rclone, others
+may require different backends
+"""
+from abc import ABC, abstractmethod
+from selfprivacy_api.backup.backuppers import AbstractBackupper
+from selfprivacy_api.graphql.queries.providers import (
+    BackupProvider as BackupProviderEnum,
+)
+
+
+class AbstractBackupProvider(ABC):
+    backupper: AbstractBackupper
+
+    name: BackupProviderEnum
+
+    def __init__(self, login="", key="", location="", repo_id=""):
+        self.backupper.set_creds(login, key, location)
+        self.login = login
+        self.key = key
+        self.location = location
+        # We do not need to do anything with this one
+        # Just remember in case the app forgets
+        self.repo_id = repo_id

selfprivacy_api/backup/storage.py

@@ -0,0 +1,198 @@
+"""
+Module for storing backup related data in redis.
+"""
+from typing import List, Optional
+from datetime import datetime
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.models.backup.provider import BackupProviderModel
+from selfprivacy_api.graphql.common_types.backup import (
+    AutobackupQuotas,
+    _AutobackupQuotas,
+)
+
+from selfprivacy_api.utils.redis_pool import RedisPool
+from selfprivacy_api.utils.redis_model_storage import (
+    store_model_as_hash,
+    hash_as_model,
+)
+
+from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
+from selfprivacy_api.backup.providers import get_kind
+
+REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:"
+REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:"
+REDIS_INITTED_CACHE = "backups:repo_initted"
+
+REDIS_PROVIDER_KEY = "backups:provider"
+REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period"
+
+REDIS_AUTOBACKUP_QUOTAS_KEY = "backups:autobackup_quotas_key"
+
+redis = RedisPool().get_connection()
+
+
+class Storage:
+    """Static class for storing backup related data in redis"""
+
+    @staticmethod
+    def reset() -> None:
+        """Deletes all backup related data from redis"""
+        redis.delete(REDIS_PROVIDER_KEY)
+        redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
+        redis.delete(REDIS_INITTED_CACHE)
+        redis.delete(REDIS_AUTOBACKUP_QUOTAS_KEY)
+
+        prefixes_to_clean = [
+            REDIS_SNAPSHOTS_PREFIX,
+            REDIS_LAST_BACKUP_PREFIX,
+        ]
+
+        for prefix in prefixes_to_clean:
+            for key in redis.keys(prefix + "*"):
+                redis.delete(key)
+
+    @staticmethod
+    def invalidate_snapshot_storage() -> None:
+        """Deletes all cached snapshots from redis"""
+        for key in redis.keys(REDIS_SNAPSHOTS_PREFIX + "*"):
+            redis.delete(key)
+
+    @staticmethod
+    def __last_backup_key(service_id: str) -> str:
+        return REDIS_LAST_BACKUP_PREFIX + service_id
+
+    @staticmethod
+    def __snapshot_key(snapshot: Snapshot) -> str:
+        return REDIS_SNAPSHOTS_PREFIX + snapshot.id
+
+    @staticmethod
+    def get_last_backup_time(service_id: str) -> Optional[datetime]:
+        """Returns last backup time for a service or None if it was never backed up"""
+        key = Storage.__last_backup_key(service_id)
+        if not redis.exists(key):
+            return None
+
+        snapshot = hash_as_model(redis, key, Snapshot)
+        if not snapshot:
+            return None
+        return snapshot.created_at
+
+    @staticmethod
+    def store_last_timestamp(service_id: str, snapshot: Snapshot) -> None:
+        """Stores last backup time for a service"""
+        store_model_as_hash(
+            redis,
+            Storage.__last_backup_key(service_id),
+            snapshot,
+        )
+
+    @staticmethod
+    def cache_snapshot(snapshot: Snapshot) -> None:
+        """Stores snapshot metadata in redis for caching purposes"""
+        snapshot_key = Storage.__snapshot_key(snapshot)
+        store_model_as_hash(redis, snapshot_key, snapshot)
+
+    @staticmethod
+    def delete_cached_snapshot(snapshot: Snapshot) -> None:
+        """Deletes snapshot metadata from redis"""
+        snapshot_key = Storage.__snapshot_key(snapshot)
+        redis.delete(snapshot_key)
+
+    @staticmethod
+    def get_cached_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]:
+        """Returns cached snapshot by id or None if it doesn't exist"""
+        key = REDIS_SNAPSHOTS_PREFIX + snapshot_id
+        if not redis.exists(key):
+            return None
+        return hash_as_model(redis, key, Snapshot)
+
+    @staticmethod
+    def get_cached_snapshots() -> List[Snapshot]:
+        """Returns all cached snapshots stored in redis"""
+        keys: list[str] = redis.keys(REDIS_SNAPSHOTS_PREFIX + "*")  # type: ignore
+        result: list[Snapshot] = []
+
+        for key in keys:
+            snapshot = hash_as_model(redis, key, Snapshot)
+            if snapshot:
+                result.append(snapshot)
+        return result
+
+    @staticmethod
+    def autobackup_period_minutes() -> Optional[int]:
+        """None means autobackup is disabled"""
+        if not redis.exists(REDIS_AUTOBACKUP_PERIOD_KEY):
+            return None
+        return int(redis.get(REDIS_AUTOBACKUP_PERIOD_KEY))  # type: ignore
+
+    @staticmethod
+    def store_autobackup_period_minutes(minutes: int) -> None:
+        """Set the new autobackup period in minutes"""
+        redis.set(REDIS_AUTOBACKUP_PERIOD_KEY, minutes)
+
+    @staticmethod
+    def delete_backup_period() -> None:
+        """Set the autobackup period to none, effectively disabling autobackup"""
+        redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
+
+    @staticmethod
+    def store_provider(provider: AbstractBackupProvider) -> None:
+        """Stores backup provider auth data in redis"""
+        model = BackupProviderModel(
+            kind=get_kind(provider),
+            login=provider.login,
+            key=provider.key,
+            location=provider.location,
+            repo_id=provider.repo_id,
+        )
+        store_model_as_hash(redis, REDIS_PROVIDER_KEY, model)
+        if Storage.load_provider() != model:
+            raise IOError("could not store the provider model: ", model.dict)
+
+    @staticmethod
+    def load_provider() -> Optional[BackupProviderModel]:
+        """Loads backup storage provider auth data from redis"""
+        provider_model = hash_as_model(
+            redis,
+            REDIS_PROVIDER_KEY,
+            BackupProviderModel,
+        )
+        return provider_model
+
+    @staticmethod
+    def has_init_mark() -> bool:
+        """Returns True if the repository was initialized"""
+        if redis.exists(REDIS_INITTED_CACHE):
+            return True
+        return False
+
+    @staticmethod
+    def mark_as_init():
+        """Marks the repository as initialized"""
+        redis.set(REDIS_INITTED_CACHE, 1)
+
+    @staticmethod
+    def mark_as_uninitted():
+        """Marks the repository as initialized"""
+        redis.delete(REDIS_INITTED_CACHE)
+
+    @staticmethod
+    def set_autobackup_quotas(quotas: AutobackupQuotas) -> None:
+        store_model_as_hash(redis, REDIS_AUTOBACKUP_QUOTAS_KEY, quotas.to_pydantic())
+
+    @staticmethod
+    def autobackup_quotas() -> AutobackupQuotas:
+        quotas_model = hash_as_model(
+            redis, REDIS_AUTOBACKUP_QUOTAS_KEY, _AutobackupQuotas
+        )
+        if quotas_model is None:
+            unlimited_quotas = AutobackupQuotas(
+                last=-1,
+                daily=-1,
+                weekly=-1,
+                monthly=-1,
+                yearly=-1,
+            )
+            return unlimited_quotas
+        return AutobackupQuotas.from_pydantic(quotas_model)  # pylint: disable=no-member

selfprivacy_api/backup/tasks.py

@@ -0,0 +1,117 @@
+"""
+The tasks module contains the worker tasks that are used to back up and restore
+"""
+from datetime import datetime, timezone
+
+from selfprivacy_api.graphql.common_types.backup import (
+    RestoreStrategy,
+    BackupReason,
+)
+
+from selfprivacy_api.models.backup.snapshot import Snapshot
+from selfprivacy_api.utils.huey import huey
+from huey import crontab
+
+from selfprivacy_api.services import get_service_by_id
+from selfprivacy_api.backup import Backups
+from selfprivacy_api.backup.jobs import add_autobackup_job
+from selfprivacy_api.jobs import Jobs, JobStatus, Job
+
+
+SNAPSHOT_CACHE_TTL_HOURS = 6
+
+
+def validate_datetime(dt: datetime) -> bool:
+    """
+    Validates that it is time to back up.
+    Also ensures that the timezone-aware time is used.
+    """
+    if dt.tzinfo is None:
+        return Backups.is_time_to_backup(dt.replace(tzinfo=timezone.utc))
+    return Backups.is_time_to_backup(dt)
+
+
+# huey tasks need to return something
+@huey.task()
+def start_backup(service_id: str, reason: BackupReason = BackupReason.EXPLICIT) -> bool:
+    """
+    The worker task that starts the backup process.
+    """
+    service = get_service_by_id(service_id)
+    if service is None:
+        raise ValueError(f"No such service: {service_id}")
+    Backups.back_up(service, reason)
+    return True
+
+
+@huey.task()
+def prune_autobackup_snapshots(job: Job) -> bool:
+    """
+    Remove all autobackup snapshots that do not fit into quotas set
+    """
+    Jobs.update(job, JobStatus.RUNNING)
+    try:
+        Backups.prune_all_autosnaps()
+    except Exception as e:
+        Jobs.update(job, JobStatus.ERROR, error=type(e).__name__ + ":" + str(e))
+        return False
+
+    Jobs.update(job, JobStatus.FINISHED)
+    return True
+
+
+@huey.task()
+def restore_snapshot(
+    snapshot: Snapshot,
+    strategy: RestoreStrategy = RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE,
+) -> bool:
+    """
+    The worker task that starts the restore process.
+    """
+    Backups.restore_snapshot(snapshot, strategy)
+    return True
+
+
+def do_autobackup() -> None:
+    """
+    Body of autobackup task, broken out to test it
+    For some reason, we cannot launch periodic huey tasks
+    inside tests
+    """
+    time = datetime.utcnow().replace(tzinfo=timezone.utc)
+    services_to_back_up = Backups.services_to_back_up(time)
+    if not services_to_back_up:
+        return
+    job = add_autobackup_job(services_to_back_up)
+
+    progress_per_service = 100 // len(services_to_back_up)
+    progress = 0
+    Jobs.update(job, JobStatus.RUNNING, progress=progress)
+
+    for service in services_to_back_up:
+        try:
+            Backups.back_up(service, BackupReason.AUTO)
+        except Exception as error:
+            Jobs.update(
+                job,
+                status=JobStatus.ERROR,
+                error=type(error).__name__ + ": " + str(error),
+            )
+            return
+        progress = progress + progress_per_service
+        Jobs.update(job, JobStatus.RUNNING, progress=progress)
+
+    Jobs.update(job, JobStatus.FINISHED)
+
+
+@huey.periodic_task(validate_datetime=validate_datetime)
+def automatic_backup() -> None:
+    """
+    The worker periodic task that starts the automatic backup process.
+    """
+    do_autobackup()
+
+
+@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
+def reload_snapshot_cache():
+    Backups.force_snapshot_cache_reload()

selfprivacy_api/backup/util.py

@@ -0,0 +1,35 @@
+import subprocess
+from os.path import exists
+from typing import Generator
+
+
+def output_yielder(command) -> Generator[str, None, None]:
+    """Note: If you break during iteration, it kills the process"""
+    with subprocess.Popen(
+        command,
+        shell=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        universal_newlines=True,
+    ) as handle:
+        if handle is None or handle.stdout is None:
+            raise ValueError("could not run command: ", command)
+
+        try:
+            for line in iter(handle.stdout.readline, ""):
+                if "NOTICE:" not in line:
+                    yield line
+        except GeneratorExit:
+            handle.kill()
+
+
+def sync(src_path: str, dest_path: str):
+    """a wrapper around rclone sync"""
+
+    if not exists(src_path):
+        raise ValueError("source dir for rclone sync must exist")
+
+    rclone_command = ["rclone", "sync", "-P", src_path, dest_path]
+    for raw_message in output_yielder(rclone_command):
+        if "ERROR" in raw_message:
+            raise ValueError(raw_message)

selfprivacy_api/dependencies.py

@@ -2,7 +2,7 @@ from fastapi import Depends, HTTPException, status
 from fastapi.security import APIKeyHeader
 from pydantic import BaseModel
 
-from selfprivacy_api.utils.auth import is_token_valid
+from selfprivacy_api.actions.api_tokens import is_token_valid
 
 
 class TokenHeader(BaseModel):
@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "2.0.9"
+    return "3.1.0"

selfprivacy_api/graphql/__init__.py

@@ -4,7 +4,7 @@ import typing
 from strawberry.permission import BasePermission
 from strawberry.types import Info
 
-from selfprivacy_api.utils.auth import is_token_valid
+from selfprivacy_api.actions.api_tokens import is_token_valid
 
 
 class IsAuthenticated(BasePermission):

selfprivacy_api/graphql/common_types/backup.py

@@ -0,0 +1,36 @@
+"""Backup"""
+# pylint: disable=too-few-public-methods
+from enum import Enum
+import strawberry
+from pydantic import BaseModel
+
+
+@strawberry.enum
+class RestoreStrategy(Enum):
+    INPLACE = "INPLACE"
+    DOWNLOAD_VERIFY_OVERWRITE = "DOWNLOAD_VERIFY_OVERWRITE"
+
+
+@strawberry.enum
+class BackupReason(Enum):
+    EXPLICIT = "EXPLICIT"
+    AUTO = "AUTO"
+    PRE_RESTORE = "PRE_RESTORE"
+
+
+class _AutobackupQuotas(BaseModel):
+    last: int
+    daily: int
+    weekly: int
+    monthly: int
+    yearly: int
+
+
+@strawberry.experimental.pydantic.type(model=_AutobackupQuotas, all_fields=True)
+class AutobackupQuotas:
+    pass
+
+
+@strawberry.experimental.pydantic.input(model=_AutobackupQuotas, all_fields=True)
+class AutobackupQuotasInput:
+    pass

selfprivacy_api/graphql/common_types/dns.py

@@ -2,6 +2,7 @@ import typing
 import strawberry
 
 
+# TODO: use https://strawberry.rocks/docs/integrations/pydantic when it is stable
 @strawberry.type
 class DnsRecord:
     """DNS record"""
@@ -11,3 +12,4 @@ class DnsRecord:
     content: str
     ttl: int
     priority: typing.Optional[int]
+    display_name: str

selfprivacy_api/graphql/common_types/jobs.py

@@ -12,6 +12,7 @@ class ApiJob:
     """Job type for GraphQL."""
 
     uid: str
+    type_id: str
     name: str
     description: str
     status: str
@@ -28,6 +29,7 @@ def job_to_api_job(job: Job) -> ApiJob:
     """Convert a Job from jobs controller to a GraphQL ApiJob."""
     return ApiJob(
         uid=str(job.uid),
+        type_id=job.type_id,
         name=job.name,
         description=job.description,
         status=job.status.name,
@@ -43,7 +45,7 @@ def job_to_api_job(job: Job) -> ApiJob:
 
 def get_api_job_by_id(job_id: str) -> typing.Optional[ApiJob]:
     """Get a job for GraphQL by its ID."""
-    job = Jobs.get_instance().get_job(job_id)
+    job = Jobs.get_job(job_id)
     if job is None:
         return None
     return job_to_api_job(job)

selfprivacy_api/graphql/common_types/service.py

@@ -1,11 +1,17 @@
 from enum import Enum
-import typing
+from typing import Optional, List
+import datetime
 import strawberry
+
+from selfprivacy_api.graphql.common_types.backup import BackupReason
 from selfprivacy_api.graphql.common_types.dns import DnsRecord
 
 from selfprivacy_api.services import get_service_by_id, get_services_by_location
 from selfprivacy_api.services import Service as ServiceInterface
+from selfprivacy_api.services import ServiceDnsRecord
+
 from selfprivacy_api.utils.block_devices import BlockDevices
+from selfprivacy_api.utils.network import get_ip4, get_ip6
 
 
 def get_usages(root: "StorageVolume") -> list["StorageUsageInterface"]:
@@ -15,7 +21,7 @@ def get_usages(root: "StorageVolume") -> list["StorageUsageInterface"]:
             service=service_to_graphql_service(service),
             title=service.get_display_name(),
             used_space=str(service.get_storage_usage()),
-            volume=get_volume_by_id(service.get_location()),
+            volume=get_volume_by_id(service.get_drive()),
         )
         for service in get_services_by_location(root.name)
     ]
@@ -30,8 +36,8 @@ class StorageVolume:
     used_space: str
     root: bool
     name: str
-    model: typing.Optional[str]
-    serial: typing.Optional[str]
+    model: Optional[str]
+    serial: Optional[str]
     type: str
 
     @strawberry.field
@@ -43,7 +49,7 @@ class StorageVolume:
 @strawberry.interface
 class StorageUsageInterface:
     used_space: str
-    volume: typing.Optional[StorageVolume]
+    volume: Optional[StorageVolume]
     title: str
 
 
@@ -51,7 +57,7 @@ class StorageUsageInterface:
 class ServiceStorageUsage(StorageUsageInterface):
     """Storage usage for a service"""
 
-    service: typing.Optional["Service"]
+    service: Optional["Service"]
 
 
 @strawberry.enum
@@ -79,7 +85,21 @@ def get_storage_usage(root: "Service") -> ServiceStorageUsage:
         service=service_to_graphql_service(service),
         title=service.get_display_name(),
         used_space=str(service.get_storage_usage()),
-        volume=get_volume_by_id(service.get_location()),
+        volume=get_volume_by_id(service.get_drive()),
+    )
+
+
+# TODO: This won't be needed when deriving DnsRecord via strawberry pydantic integration
+# https://strawberry.rocks/docs/integrations/pydantic
+# Remove when the link above says it got stable.
+def service_dns_to_graphql(record: ServiceDnsRecord) -> DnsRecord:
+    return DnsRecord(
+        record_type=record.type,
+        name=record.name,
+        content=record.content,
+        ttl=record.ttl,
+        priority=record.priority,
+        display_name=record.display_name,
     )
 
 
@@ -92,15 +112,39 @@ class Service:
     is_movable: bool
     is_required: bool
     is_enabled: bool
+    can_be_backed_up: bool
+    backup_description: str
     status: ServiceStatusEnum
-    url: typing.Optional[str]
-    dns_records: typing.Optional[typing.List[DnsRecord]]
+    url: Optional[str]
+
+    @strawberry.field
+    def dns_records(self) -> Optional[List[DnsRecord]]:
+        service = get_service_by_id(self.id)
+        if service is None:
+            raise LookupError(f"no service {self.id}. Should be unreachable")
+
+        raw_records = service.get_dns_records(get_ip4(), get_ip6())
+        dns_records = [service_dns_to_graphql(record) for record in raw_records]
+        return dns_records
 
     @strawberry.field
     def storage_usage(self) -> ServiceStorageUsage:
         """Get storage usage for a service"""
         return get_storage_usage(self)
 
+    # TODO: fill this
+    @strawberry.field
+    def backup_snapshots(self) -> Optional[List["SnapshotInfo"]]:
+        return None
+
+
+@strawberry.type
+class SnapshotInfo:
+    id: str
+    service: Service
+    created_at: datetime.datetime
+    reason: BackupReason
+
 
 def service_to_graphql_service(service: ServiceInterface) -> Service:
     """Convert service to graphql service"""
@@ -112,22 +156,14 @@ def service_to_graphql_service(service: ServiceInterface) -> Service:
         is_movable=service.is_movable(),
         is_required=service.is_required(),
         is_enabled=service.is_enabled(),
+        can_be_backed_up=service.can_be_backed_up(),
+        backup_description=service.get_backup_description(),
         status=ServiceStatusEnum(service.get_status().value),
         url=service.get_url(),
-        dns_records=[
-            DnsRecord(
-                record_type=record.type,
-                name=record.name,
-                content=record.content,
-                ttl=record.ttl,
-                priority=record.priority,
-            )
-            for record in service.get_dns_records()
-        ],
     )
 
 
-def get_volume_by_id(volume_id: str) -> typing.Optional[StorageVolume]:
+def get_volume_by_id(volume_id: str) -> Optional[StorageVolume]:
     """Get volume by id"""
     volume = BlockDevices().get_block_device(volume_id)
     if volume is None:

selfprivacy_api/graphql/common_types/user.py

@@ -17,7 +17,6 @@ class UserType(Enum):
 
 @strawberry.type
 class User:
-
     user_type: UserType
     username: str
     # userHomeFolderspace: UserHomeFolderUsage
@@ -32,7 +31,6 @@ class UserMutationReturn(MutationReturnInterface):
 
 
 def get_user_by_username(username: str) -> typing.Optional[User]:
-
     user = users_actions.get_user_by_username(username)
     if user is None:
         return None

selfprivacy_api/graphql/mutations/api_mutations.py

@@ -11,6 +11,11 @@ from selfprivacy_api.actions.api_tokens import (
     NotFoundException,
     delete_api_token,
     get_new_api_recovery_key,
+    use_mnemonic_recovery_token,
+    refresh_api_token,
+    delete_new_device_auth_token,
+    get_new_device_auth_token,
+    use_new_device_auth_token,
 )
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.mutation_interface import (
@@ -18,14 +23,6 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
     MutationReturnInterface,
 )
 
-from selfprivacy_api.utils.auth import (
-    delete_new_device_auth_token,
-    get_new_device_auth_token,
-    refresh_token,
-    use_mnemonic_recoverery_token,
-    use_new_device_auth_token,
-)
-
 
 @strawberry.type
 class ApiKeyMutationReturn(MutationReturnInterface):
@@ -98,50 +95,53 @@ class ApiMutations:
         self, input: UseRecoveryKeyInput
     ) -> DeviceApiTokenMutationReturn:
         """Use recovery key"""
-        token = use_mnemonic_recoverery_token(input.key, input.deviceName)
-        if token is None:
+        token = use_mnemonic_recovery_token(input.key, input.deviceName)
+        if token is not None:
+            return DeviceApiTokenMutationReturn(
+                success=True,
+                message="Recovery key used",
+                code=200,
+                token=token,
+            )
+        else:
             return DeviceApiTokenMutationReturn(
                 success=False,
                 message="Recovery key not found",
                 code=404,
                 token=None,
             )
-        return DeviceApiTokenMutationReturn(
-            success=True,
-            message="Recovery key used",
-            code=200,
-            token=token,
-        )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def refresh_device_api_token(self, info: Info) -> DeviceApiTokenMutationReturn:
         """Refresh device api token"""
-        token = (
+        token_string = (
             info.context["request"]
             .headers.get("Authorization", "")
             .replace("Bearer ", "")
         )
-        if token is None:
+        if token_string is None:
             return DeviceApiTokenMutationReturn(
                 success=False,
                 message="Token not found",
                 code=404,
                 token=None,
             )
-        new_token = refresh_token(token)
-        if new_token is None:
+
+        try:
+            new_token = refresh_api_token(token_string)
+            return DeviceApiTokenMutationReturn(
+                success=True,
+                message="Token refreshed",
+                code=200,
+                token=new_token,
+            )
+        except NotFoundException:
             return DeviceApiTokenMutationReturn(
                 success=False,
                 message="Token not found",
                 code=404,
                 token=None,
             )
-        return DeviceApiTokenMutationReturn(
-            success=True,
-            message="Token refreshed",
-            code=200,
-            token=new_token,
-        )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def delete_device_api_token(self, device: str, info: Info) -> GenericMutationReturn:

selfprivacy_api/graphql/mutations/backup_mutations.py

@@ -0,0 +1,241 @@
+import typing
+import strawberry
+
+from selfprivacy_api.jobs import Jobs
+
+from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.graphql.mutations.mutation_interface import (
+    GenericMutationReturn,
+    GenericJobMutationReturn,
+    MutationReturnInterface,
+)
+from selfprivacy_api.graphql.queries.backup import BackupConfiguration
+from selfprivacy_api.graphql.queries.backup import Backup
+from selfprivacy_api.graphql.queries.providers import BackupProvider
+from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
+from selfprivacy_api.graphql.common_types.backup import (
+    AutobackupQuotasInput,
+    RestoreStrategy,
+)
+
+from selfprivacy_api.backup import Backups
+from selfprivacy_api.services import get_service_by_id
+from selfprivacy_api.backup.tasks import (
+    start_backup,
+    restore_snapshot,
+    prune_autobackup_snapshots,
+)
+from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job
+
+
+@strawberry.input
+class InitializeRepositoryInput:
+    """Initialize repository input"""
+
+    provider: BackupProvider
+    # The following field may become optional for other providers?
+    # Backblaze takes bucket id and name
+    location_id: str
+    location_name: str
+    # Key ID and key for Backblaze
+    login: str
+    password: str
+
+
+@strawberry.type
+class GenericBackupConfigReturn(MutationReturnInterface):
+    """Generic backup config return"""
+
+    configuration: typing.Optional[BackupConfiguration]
+
+
+@strawberry.type
+class BackupMutations:
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def initialize_repository(
+        self, repository: InitializeRepositoryInput
+    ) -> GenericBackupConfigReturn:
+        """Initialize a new repository"""
+        Backups.set_provider(
+            kind=repository.provider,
+            login=repository.login,
+            key=repository.password,
+            location=repository.location_name,
+            repo_id=repository.location_id,
+        )
+        Backups.init_repo()
+        return GenericBackupConfigReturn(
+            success=True,
+            message="",
+            code=200,
+            configuration=Backup().configuration(),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def remove_repository(self) -> GenericBackupConfigReturn:
+        """Remove repository"""
+        Backups.reset()
+        return GenericBackupConfigReturn(
+            success=True,
+            message="",
+            code=200,
+            configuration=Backup().configuration(),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def set_autobackup_period(
+        self, period: typing.Optional[int] = None
+    ) -> GenericBackupConfigReturn:
+        """Set autobackup period. None is to disable autobackup"""
+        if period is not None:
+            Backups.set_autobackup_period_minutes(period)
+        else:
+            Backups.set_autobackup_period_minutes(0)
+
+        return GenericBackupConfigReturn(
+            success=True,
+            message="",
+            code=200,
+            configuration=Backup().configuration(),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def set_autobackup_quotas(
+        self, quotas: AutobackupQuotasInput
+    ) -> GenericBackupConfigReturn:
+        """
+        Set autobackup quotas.
+        Values <=0 for any timeframe mean no limits for that timeframe.
+        To disable autobackup use autobackup period setting, not this mutation.
+        """
+
+        job = Jobs.add(
+            name="Trimming autobackup snapshots",
+            type_id="backups.autobackup_trimming",
+            description="Pruning the excessive snapshots after the new autobackup quotas are set",
+        )
+
+        try:
+            Backups.set_autobackup_quotas(quotas)
+            # this task is async and can fail with only a job to report the error
+            prune_autobackup_snapshots(job)
+            return GenericBackupConfigReturn(
+                success=True,
+                message="",
+                code=200,
+                configuration=Backup().configuration(),
+            )
+
+        except Exception as e:
+            return GenericBackupConfigReturn(
+                success=False,
+                message=type(e).__name__ + ":" + str(e),
+                code=400,
+                configuration=Backup().configuration(),
+            )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def start_backup(self, service_id: str) -> GenericJobMutationReturn:
+        """Start backup"""
+
+        service = get_service_by_id(service_id)
+        if service is None:
+            return GenericJobMutationReturn(
+                success=False,
+                code=300,
+                message=f"nonexistent service: {service_id}",
+                job=None,
+            )
+
+        job = add_backup_job(service)
+        start_backup(service_id)
+
+        return GenericJobMutationReturn(
+            success=True,
+            code=200,
+            message="Backup job queued",
+            job=job_to_api_job(job),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def restore_backup(
+        self,
+        snapshot_id: str,
+        strategy: RestoreStrategy = RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE,
+    ) -> GenericJobMutationReturn:
+        """Restore backup"""
+        snap = Backups.get_snapshot_by_id(snapshot_id)
+        if snap is None:
+            return GenericJobMutationReturn(
+                success=False,
+                code=404,
+                message=f"No such snapshot: {snapshot_id}",
+                job=None,
+            )
+
+        service = get_service_by_id(snap.service_name)
+        if service is None:
+            return GenericJobMutationReturn(
+                success=False,
+                code=404,
+                message=f"nonexistent service: {snap.service_name}",
+                job=None,
+            )
+
+        try:
+            job = add_restore_job(snap)
+        except ValueError as error:
+            return GenericJobMutationReturn(
+                success=False,
+                code=400,
+                message=str(error),
+                job=None,
+            )
+
+        restore_snapshot(snap, strategy)
+
+        return GenericJobMutationReturn(
+            success=True,
+            code=200,
+            message="restore job created",
+            job=job_to_api_job(job),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def forget_snapshot(self, snapshot_id: str) -> GenericMutationReturn:
+        """Forget a snapshot.
+        Makes it inaccessible from the server.
+        After some time, the data (encrypted) will not be recoverable
+        from the backup server too, but not immediately"""
+
+        snap = Backups.get_snapshot_by_id(snapshot_id)
+        if snap is None:
+            return GenericMutationReturn(
+                success=False,
+                code=404,
+                message=f"snapshot {snapshot_id} not found",
+            )
+
+        try:
+            Backups.forget_snapshot(snap)
+            return GenericMutationReturn(
+                success=True,
+                code=200,
+                message="",
+            )
+        except Exception as error:
+            return GenericMutationReturn(
+                success=False,
+                code=400,
+                message=str(error),
+            )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def force_snapshots_reload(self) -> GenericMutationReturn:
+        """Force snapshots reload"""
+        Backups.force_snapshot_cache_reload()
+        return GenericMutationReturn(
+            success=True,
+            code=200,
+            message="",
+        )

selfprivacy_api/graphql/mutations/deprecated_mutations.py

@@ -0,0 +1,216 @@
+"""Deprecated mutations
+
+There was made a mistake, where mutations were not grouped, and were instead
+placed in the root of mutations schema. In this file, we import all the
+mutations from and provide them to the root for backwards compatibility.
+"""
+
+import strawberry
+from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.graphql.common_types.user import UserMutationReturn
+from selfprivacy_api.graphql.mutations.api_mutations import (
+    ApiKeyMutationReturn,
+    ApiMutations,
+    DeviceApiTokenMutationReturn,
+)
+from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
+from selfprivacy_api.graphql.mutations.job_mutations import JobMutations
+from selfprivacy_api.graphql.mutations.mutation_interface import (
+    GenericJobMutationReturn,
+    GenericMutationReturn,
+)
+from selfprivacy_api.graphql.mutations.services_mutations import (
+    ServiceJobMutationReturn,
+    ServiceMutationReturn,
+    ServicesMutations,
+)
+from selfprivacy_api.graphql.mutations.storage_mutations import StorageMutations
+from selfprivacy_api.graphql.mutations.system_mutations import (
+    AutoUpgradeSettingsMutationReturn,
+    SystemMutations,
+    TimezoneMutationReturn,
+)
+from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
+from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
+
+
+def deprecated_mutation(func, group, auth=True):
+    return strawberry.mutation(
+        resolver=func,
+        permission_classes=[IsAuthenticated] if auth else [],
+        deprecation_reason=f"Use `{group}.{func.__name__}` instead",
+    )
+
+
+@strawberry.type
+class DeprecatedApiMutations:
+    get_new_recovery_api_key: ApiKeyMutationReturn = deprecated_mutation(
+        ApiMutations.get_new_recovery_api_key,
+        "api",
+    )
+
+    use_recovery_api_key: DeviceApiTokenMutationReturn = deprecated_mutation(
+        ApiMutations.use_recovery_api_key,
+        "api",
+        auth=False,
+    )
+
+    refresh_device_api_token: DeviceApiTokenMutationReturn = deprecated_mutation(
+        ApiMutations.refresh_device_api_token,
+        "api",
+    )
+
+    delete_device_api_token: GenericMutationReturn = deprecated_mutation(
+        ApiMutations.delete_device_api_token,
+        "api",
+    )
+
+    get_new_device_api_key: ApiKeyMutationReturn = deprecated_mutation(
+        ApiMutations.get_new_device_api_key,
+        "api",
+    )
+
+    invalidate_new_device_api_key: GenericMutationReturn = deprecated_mutation(
+        ApiMutations.invalidate_new_device_api_key,
+        "api",
+    )
+
+    authorize_with_new_device_api_key: DeviceApiTokenMutationReturn = (
+        deprecated_mutation(
+            ApiMutations.authorize_with_new_device_api_key,
+            "api",
+            auth=False,
+        )
+    )
+
+
+@strawberry.type
+class DeprecatedSystemMutations:
+    change_timezone: TimezoneMutationReturn = deprecated_mutation(
+        SystemMutations.change_timezone,
+        "system",
+    )
+
+    change_auto_upgrade_settings: AutoUpgradeSettingsMutationReturn = (
+        deprecated_mutation(
+            SystemMutations.change_auto_upgrade_settings,
+            "system",
+        )
+    )
+
+    run_system_rebuild: GenericMutationReturn = deprecated_mutation(
+        SystemMutations.run_system_rebuild,
+        "system",
+    )
+
+    run_system_rollback: GenericMutationReturn = deprecated_mutation(
+        SystemMutations.run_system_rollback,
+        "system",
+    )
+
+    run_system_upgrade: GenericMutationReturn = deprecated_mutation(
+        SystemMutations.run_system_upgrade,
+        "system",
+    )
+
+    reboot_system: GenericMutationReturn = deprecated_mutation(
+        SystemMutations.reboot_system,
+        "system",
+    )
+
+    pull_repository_changes: GenericMutationReturn = deprecated_mutation(
+        SystemMutations.pull_repository_changes,
+        "system",
+    )
+
+
+@strawberry.type
+class DeprecatedUsersMutations:
+    create_user: UserMutationReturn = deprecated_mutation(
+        UsersMutations.create_user,
+        "users",
+    )
+
+    delete_user: GenericMutationReturn = deprecated_mutation(
+        UsersMutations.delete_user,
+        "users",
+    )
+
+    update_user: UserMutationReturn = deprecated_mutation(
+        UsersMutations.update_user,
+        "users",
+    )
+
+    add_ssh_key: UserMutationReturn = deprecated_mutation(
+        UsersMutations.add_ssh_key,
+        "users",
+    )
+
+    remove_ssh_key: UserMutationReturn = deprecated_mutation(
+        UsersMutations.remove_ssh_key,
+        "users",
+    )
+
+
+@strawberry.type
+class DeprecatedStorageMutations:
+    resize_volume: GenericMutationReturn = deprecated_mutation(
+        StorageMutations.resize_volume,
+        "storage",
+    )
+
+    mount_volume: GenericMutationReturn = deprecated_mutation(
+        StorageMutations.mount_volume,
+        "storage",
+    )
+
+    unmount_volume: GenericMutationReturn = deprecated_mutation(
+        StorageMutations.unmount_volume,
+        "storage",
+    )
+
+    migrate_to_binds: GenericJobMutationReturn = deprecated_mutation(
+        StorageMutations.migrate_to_binds,
+        "storage",
+    )
+
+
+@strawberry.type
+class DeprecatedServicesMutations:
+    enable_service: ServiceMutationReturn = deprecated_mutation(
+        ServicesMutations.enable_service,
+        "services",
+    )
+
+    disable_service: ServiceMutationReturn = deprecated_mutation(
+        ServicesMutations.disable_service,
+        "services",
+    )
+
+    stop_service: ServiceMutationReturn = deprecated_mutation(
+        ServicesMutations.stop_service,
+        "services",
+    )
+
+    start_service: ServiceMutationReturn = deprecated_mutation(
+        ServicesMutations.start_service,
+        "services",
+    )
+
+    restart_service: ServiceMutationReturn = deprecated_mutation(
+        ServicesMutations.restart_service,
+        "services",
+    )
+
+    move_service: ServiceJobMutationReturn = deprecated_mutation(
+        ServicesMutations.move_service,
+        "services",
+    )
+
+
+@strawberry.type
+class DeprecatedJobMutations:
+    remove_job: GenericMutationReturn = deprecated_mutation(
+        JobMutations.remove_job,
+        "jobs",
+    )

selfprivacy_api/graphql/mutations/job_mutations.py

@@ -14,7 +14,7 @@ class JobMutations:
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def remove_job(self, job_id: str) -> GenericMutationReturn:
         """Remove a job from the queue"""
-        result = Jobs.get_instance().remove_by_uid(job_id)
+        result = Jobs.remove_by_uid(job_id)
         if result:
             return GenericMutationReturn(
                 success=True,

selfprivacy_api/graphql/mutations/mutation_interface.py

@@ -17,5 +17,5 @@ class GenericMutationReturn(MutationReturnInterface):
 
 
 @strawberry.type
-class GenericJobButationReturn(MutationReturnInterface):
+class GenericJobMutationReturn(MutationReturnInterface):
     job: typing.Optional[ApiJob] = None

selfprivacy_api/graphql/mutations/services_mutations.py

@@ -4,18 +4,26 @@ import typing
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
+from selfprivacy_api.jobs import JobStatus
 
+from traceback import format_tb as format_traceback
+
+from selfprivacy_api.graphql.mutations.mutation_interface import (
+    GenericJobMutationReturn,
+    GenericMutationReturn,
+)
 from selfprivacy_api.graphql.common_types.service import (
     Service,
     service_to_graphql_service,
 )
-from selfprivacy_api.graphql.mutations.mutation_interface import (
-    GenericJobButationReturn,
-    GenericMutationReturn,
+
+from selfprivacy_api.actions.services import (
+    move_service,
+    ServiceNotFoundError,
+    VolumeNotFoundError,
 )
 
 from selfprivacy_api.services import get_service_by_id
-from selfprivacy_api.utils.block_devices import BlockDevices
 
 
 @strawberry.type
@@ -34,7 +42,7 @@ class MoveServiceInput:
 
 
 @strawberry.type
-class ServiceJobMutationReturn(GenericJobButationReturn):
+class ServiceJobMutationReturn(GenericJobMutationReturn):
     """Service job mutation return type."""
 
     service: typing.Optional[Service] = None
@@ -47,14 +55,22 @@ class ServicesMutations:
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def enable_service(self, service_id: str) -> ServiceMutationReturn:
         """Enable service."""
-        service = get_service_by_id(service_id)
-        if service is None:
+        try:
+            service = get_service_by_id(service_id)
+            if service is None:
+                return ServiceMutationReturn(
+                    success=False,
+                    message="Service not found.",
+                    code=404,
+                )
+            service.enable()
+        except Exception as e:
             return ServiceMutationReturn(
                 success=False,
-                message="Service not found.",
-                code=404,
+                message=pretty_error(e),
+                code=400,
             )
-        service.enable()
+
         return ServiceMutationReturn(
             success=True,
             message="Service enabled.",
@@ -65,14 +81,21 @@ class ServicesMutations:
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def disable_service(self, service_id: str) -> ServiceMutationReturn:
         """Disable service."""
-        service = get_service_by_id(service_id)
-        if service is None:
+        try:
+            service = get_service_by_id(service_id)
+            if service is None:
+                return ServiceMutationReturn(
+                    success=False,
+                    message="Service not found.",
+                    code=404,
+                )
+            service.disable()
+        except Exception as e:
             return ServiceMutationReturn(
                 success=False,
-                message="Service not found.",
-                code=404,
+                message=pretty_error(e),
+                code=400,
             )
-        service.disable()
         return ServiceMutationReturn(
             success=True,
             message="Service disabled.",
@@ -137,33 +160,58 @@ class ServicesMutations:
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def move_service(self, input: MoveServiceInput) -> ServiceJobMutationReturn:
         """Move service."""
+        # We need a service instance for a reply later
         service = get_service_by_id(input.service_id)
         if service is None:
             return ServiceJobMutationReturn(
                 success=False,
-                message="Service not found.",
+                message=f"Service does not exist: {input.service_id}",
                 code=404,
             )
-        if not service.is_movable():
+
+        try:
+            job = move_service(input.service_id, input.location)
+
+        except (ServiceNotFoundError, VolumeNotFoundError) as e:
             return ServiceJobMutationReturn(
                 success=False,
-                message="Service is not movable.",
+                message=pretty_error(e),
+                code=404,
+            )
+        except Exception as e:
+            return ServiceJobMutationReturn(
+                success=False,
+                message=pretty_error(e),
                 code=400,
                 service=service_to_graphql_service(service),
             )
-        volume = BlockDevices().get_block_device(input.location)
-        if volume is None:
+
+        if job.status in [JobStatus.CREATED, JobStatus.RUNNING]:
+            return ServiceJobMutationReturn(
+                success=True,
+                message="Started moving the service.",
+                code=200,
+                service=service_to_graphql_service(service),
+                job=job_to_api_job(job),
+            )
+        elif job.status == JobStatus.FINISHED:
+            return ServiceJobMutationReturn(
+                success=True,
+                message="Service moved.",
+                code=200,
+                service=service_to_graphql_service(service),
+                job=job_to_api_job(job),
+            )
+        else:
             return ServiceJobMutationReturn(
                 success=False,
-                message="Volume not found.",
-                code=404,
+                message=f"While moving service and performing the step '{job.status_text}', error occured: {job.error}",
+                code=400,
                 service=service_to_graphql_service(service),
+                job=job_to_api_job(job),
             )
-        job = service.move_to_volume(volume)
-        return ServiceJobMutationReturn(
-            success=True,
-            message="Service moved.",
-            code=200,
-            service=service_to_graphql_service(service),
-            job=job_to_api_job(job),
-        )
+
+
+def pretty_error(e: Exception) -> str:
+    traceback = "/r".join(format_traceback(e.__traceback__))
+    return type(e).__name__ + ": " + str(e) + ": " + traceback

selfprivacy_api/graphql/mutations/ssh_mutations.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-"""Users management module"""
-# pylint: disable=too-few-public-methods
-
-import strawberry
-from selfprivacy_api.actions.users import UserNotFound
-
-from selfprivacy_api.graphql import IsAuthenticated
-from selfprivacy_api.actions.ssh import (
-    InvalidPublicKey,
-    KeyAlreadyExists,
-    KeyNotFound,
-    create_ssh_key,
-    remove_ssh_key,
-)
-from selfprivacy_api.graphql.common_types.user import (
-    UserMutationReturn,
-    get_user_by_username,
-)
-
-
-@strawberry.input
-class SshMutationInput:
-    """Input type for ssh mutation"""
-
-    username: str
-    ssh_key: str
-
-
-@strawberry.type
-class SshMutations:
-    """Mutations ssh"""
-
-    @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def add_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
-        """Add a new ssh key"""
-
-        try:
-            create_ssh_key(ssh_input.username, ssh_input.ssh_key)
-        except KeyAlreadyExists:
-            return UserMutationReturn(
-                success=False,
-                message="Key already exists",
-                code=409,
-            )
-        except InvalidPublicKey:
-            return UserMutationReturn(
-                success=False,
-                message="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
-                code=400,
-            )
-        except UserNotFound:
-            return UserMutationReturn(
-                success=False,
-                message="User not found",
-                code=404,
-            )
-        except Exception as e:
-            return UserMutationReturn(
-                success=False,
-                message=str(e),
-                code=500,
-            )
-
-        return UserMutationReturn(
-            success=True,
-            message="New SSH key successfully written",
-            code=201,
-            user=get_user_by_username(ssh_input.username),
-        )
-
-    @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def remove_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
-        """Remove ssh key from user"""
-
-        try:
-            remove_ssh_key(ssh_input.username, ssh_input.ssh_key)
-        except KeyNotFound:
-            return UserMutationReturn(
-                success=False,
-                message="Key not found",
-                code=404,
-            )
-        except UserNotFound:
-            return UserMutationReturn(
-                success=False,
-                message="User not found",
-                code=404,
-            )
-        except Exception as e:
-            return UserMutationReturn(
-                success=False,
-                message=str(e),
-                code=500,
-            )
-
-        return UserMutationReturn(
-            success=True,
-            message="SSH key successfully removed",
-            code=200,
-            user=get_user_by_username(ssh_input.username),
-        )

selfprivacy_api/graphql/mutations/storage_mutations.py

@@ -4,7 +4,7 @@ from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
 from selfprivacy_api.utils.block_devices import BlockDevices
 from selfprivacy_api.graphql.mutations.mutation_interface import (
-    GenericJobButationReturn,
+    GenericJobMutationReturn,
     GenericMutationReturn,
 )
 from selfprivacy_api.jobs.migrate_to_binds import (
@@ -79,10 +79,10 @@ class StorageMutations:
         )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def migrate_to_binds(self, input: MigrateToBindsInput) -> GenericJobButationReturn:
+    def migrate_to_binds(self, input: MigrateToBindsInput) -> GenericJobMutationReturn:
         """Migrate to binds"""
         if is_bind_migrated():
-            return GenericJobButationReturn(
+            return GenericJobMutationReturn(
                 success=False, code=409, message="Already migrated to binds"
             )
         job = start_bind_migration(
@@ -94,7 +94,7 @@ class StorageMutations:
                 pleroma_block_device=input.pleroma_block_device,
             )
         )
-        return GenericJobButationReturn(
+        return GenericJobMutationReturn(
             success=True,
             code=200,
             message="Migration to binds started, rebuild the system to apply changes",

selfprivacy_api/graphql/mutations/system_mutations.py

@@ -3,12 +3,18 @@
 import typing
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
 from selfprivacy_api.graphql.mutations.mutation_interface import (
+    GenericJobMutationReturn,
     GenericMutationReturn,
     MutationReturnInterface,
+    GenericJobMutationReturn,
 )
 
 import selfprivacy_api.actions.system as system_actions
+from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
+from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
+import selfprivacy_api.actions.ssh as ssh_actions
 
 
 @strawberry.type
@@ -26,6 +32,22 @@ class AutoUpgradeSettingsMutationReturn(MutationReturnInterface):
     allowReboot: bool
 
 
+@strawberry.type
+class SSHSettingsMutationReturn(MutationReturnInterface):
+    """A return type for after changing SSH settings"""
+
+    enable: bool
+    password_authentication: bool
+
+
+@strawberry.input
+class SSHSettingsInput:
+    """Input type for SSH settings"""
+
+    enable: bool
+    password_authentication: bool
+
+
 @strawberry.input
 class AutoUpgradeSettingsInput:
     """Input type for auto upgrade settings"""
@@ -77,40 +99,90 @@ class SystemMutations:
         )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def run_system_rebuild(self) -> GenericMutationReturn:
-        system_actions.rebuild_system()
-        return GenericMutationReturn(
-            success=True,
-            message="Starting rebuild system",
-            code=200,
+    def change_ssh_settings(
+        self, settings: SSHSettingsInput
+    ) -> SSHSettingsMutationReturn:
+        """Change ssh settings of the server."""
+        ssh_actions.set_ssh_settings(
+            enable=settings.enable,
+            password_authentication=settings.password_authentication,
         )
 
+        new_settings = ssh_actions.get_ssh_settings()
+
+        return SSHSettingsMutationReturn(
+            success=True,
+            message="SSH settings changed",
+            code=200,
+            enable=new_settings.enable,
+            password_authentication=new_settings.passwordAuthentication,
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def run_system_rebuild(self) -> GenericJobMutationReturn:
+        try:
+            job = system_actions.rebuild_system()
+            return GenericJobMutationReturn(
+                success=True,
+                message="Starting system rebuild",
+                code=200,
+                job=job_to_api_job(job),
+            )
+        except system_actions.ShellException as e:
+            return GenericJobMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
+
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def run_system_rollback(self) -> GenericMutationReturn:
         system_actions.rollback_system()
-        return GenericMutationReturn(
-            success=True,
-            message="Starting rebuild system",
-            code=200,
-        )
+        try:
+            return GenericMutationReturn(
+                success=True,
+                message="Starting system rollback",
+                code=200,
+            )
+        except system_actions.ShellException as e:
+            return GenericMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def run_system_upgrade(self) -> GenericMutationReturn:
-        system_actions.upgrade_system()
-        return GenericMutationReturn(
-            success=True,
-            message="Starting rebuild system",
-            code=200,
-        )
+    def run_system_upgrade(self) -> GenericJobMutationReturn:
+        try:
+            job = system_actions.upgrade_system()
+            return GenericJobMutationReturn(
+                success=True,
+                message="Starting system upgrade",
+                code=200,
+                job=job_to_api_job(job),
+            )
+        except system_actions.ShellException as e:
+            return GenericJobMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def reboot_system(self) -> GenericMutationReturn:
         system_actions.reboot_system()
-        return GenericMutationReturn(
-            success=True,
-            message="System reboot has started",
-            code=200,
-        )
+        try:
+            return GenericMutationReturn(
+                success=True,
+                message="System reboot has started",
+                code=200,
+            )
+        except system_actions.ShellException as e:
+            return GenericMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def pull_repository_changes(self) -> GenericMutationReturn:
@@ -126,3 +198,14 @@ class SystemMutations:
             message=f"Failed to pull repository changes:\n{result.data}",
             code=500,
         )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def nix_collect_garbage(self) -> GenericJobMutationReturn:
+        job = start_nix_collect_garbage()
+
+        return GenericJobMutationReturn(
+            success=True,
+            code=200,
+            message="Garbage collector started...",
+            job=job_to_api_job(job),
+        )

selfprivacy_api/graphql/mutations/users_mutations.py

@@ -3,10 +3,18 @@
 # pylint: disable=too-few-public-methods
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.actions.users import UserNotFound
 from selfprivacy_api.graphql.common_types.user import (
     UserMutationReturn,
     get_user_by_username,
 )
+from selfprivacy_api.actions.ssh import (
+    InvalidPublicKey,
+    KeyAlreadyExists,
+    KeyNotFound,
+    create_ssh_key,
+    remove_ssh_key,
+)
 from selfprivacy_api.graphql.mutations.mutation_interface import (
     GenericMutationReturn,
 )
@@ -21,8 +29,16 @@ class UserMutationInput:
     password: str
 
 
+@strawberry.input
+class SshMutationInput:
+    """Input type for ssh mutation"""
+
+    username: str
+    ssh_key: str
+
+
 @strawberry.type
-class UserMutations:
+class UsersMutations:
     """Mutations change user settings"""
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
@@ -53,6 +69,12 @@ class UserMutations:
                 message=str(e),
                 code=400,
             )
+        except users_actions.InvalidConfiguration as e:
+            return UserMutationReturn(
+                success=False,
+                message=str(e),
+                code=400,
+            )
         except users_actions.UserAlreadyExists as e:
             return UserMutationReturn(
                 success=False,
@@ -115,3 +137,73 @@ class UserMutations:
             code=200,
             user=get_user_by_username(user.username),
         )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def add_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
+        """Add a new ssh key"""
+
+        try:
+            create_ssh_key(ssh_input.username, ssh_input.ssh_key)
+        except KeyAlreadyExists:
+            return UserMutationReturn(
+                success=False,
+                message="Key already exists",
+                code=409,
+            )
+        except InvalidPublicKey:
+            return UserMutationReturn(
+                success=False,
+                message="Invalid key type. Only ssh-ed25519, ssh-rsa and ecdsa are supported",
+                code=400,
+            )
+        except UserNotFound:
+            return UserMutationReturn(
+                success=False,
+                message="User not found",
+                code=404,
+            )
+        except Exception as e:
+            return UserMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
+
+        return UserMutationReturn(
+            success=True,
+            message="New SSH key successfully written",
+            code=201,
+            user=get_user_by_username(ssh_input.username),
+        )
+
+    @strawberry.mutation(permission_classes=[IsAuthenticated])
+    def remove_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
+        """Remove ssh key from user"""
+
+        try:
+            remove_ssh_key(ssh_input.username, ssh_input.ssh_key)
+        except KeyNotFound:
+            return UserMutationReturn(
+                success=False,
+                message="Key not found",
+                code=404,
+            )
+        except UserNotFound:
+            return UserMutationReturn(
+                success=False,
+                message="User not found",
+                code=404,
+            )
+        except Exception as e:
+            return UserMutationReturn(
+                success=False,
+                message=str(e),
+                code=500,
+            )
+
+        return UserMutationReturn(
+            success=True,
+            message="SSH key successfully removed",
+            code=200,
+            user=get_user_by_username(ssh_input.username),
+        )

selfprivacy_api/graphql/queries/api_queries.py

@@ -4,16 +4,12 @@ import datetime
 import typing
 import strawberry
 from strawberry.types import Info
-from selfprivacy_api.actions.api_tokens import get_api_tokens_with_caller_flag
-from selfprivacy_api.graphql import IsAuthenticated
-from selfprivacy_api.utils import parse_date
-from selfprivacy_api.dependencies import get_api_version as get_api_version_dependency
-
-from selfprivacy_api.utils.auth import (
-    get_recovery_token_status,
-    is_recovery_token_exists,
-    is_recovery_token_valid,
+from selfprivacy_api.actions.api_tokens import (
+    get_api_tokens_with_caller_flag,
+    get_api_recovery_token_status,
 )
+from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.dependencies import get_api_version as get_api_version_dependency
 
 
 def get_api_version() -> str:
@@ -42,17 +38,9 @@ class ApiRecoveryKeyStatus:
 
 
 def get_recovery_key_status() -> ApiRecoveryKeyStatus:
-    """Get recovery key status"""
-    if not is_recovery_token_exists():
-        return ApiRecoveryKeyStatus(
-            exists=False,
-            valid=False,
-            creation_date=None,
-            expiration_date=None,
-            uses_left=None,
-        )
-    status = get_recovery_token_status()
-    if status is None:
+    """Get recovery key status, times are timezone-aware"""
+    status = get_api_recovery_token_status()
+    if status is None or not status.exists:
         return ApiRecoveryKeyStatus(
             exists=False,
             valid=False,
@@ -62,12 +50,10 @@ def get_recovery_key_status() -> ApiRecoveryKeyStatus:
         )
     return ApiRecoveryKeyStatus(
         exists=True,
-        valid=is_recovery_token_valid(),
-        creation_date=parse_date(status["date"]),
-        expiration_date=parse_date(status["expiration"])
-        if status["expiration"] is not None
-        else None,
-        uses_left=status["uses_left"] if status["uses_left"] is not None else None,
+        valid=status.valid,
+        creation_date=status.date,
+        expiration_date=status.expiration,
+        uses_left=status.uses_left,
     )
 
 

selfprivacy_api/graphql/queries/backup.py

@@ -0,0 +1,83 @@
+"""Backup"""
+# pylint: disable=too-few-public-methods
+import typing
+import strawberry
+
+
+from selfprivacy_api.backup import Backups
+from selfprivacy_api.backup.local_secret import LocalBackupSecret
+from selfprivacy_api.graphql.queries.providers import BackupProvider
+from selfprivacy_api.graphql.common_types.service import (
+    Service,
+    ServiceStatusEnum,
+    SnapshotInfo,
+    service_to_graphql_service,
+)
+from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas
+from selfprivacy_api.services import get_service_by_id
+
+
+@strawberry.type
+class BackupConfiguration:
+    provider: BackupProvider
+    # When server is lost, the app should have the key to decrypt backups
+    # on a new server
+    encryption_key: str
+    # False when repo is not initialized and not ready to be used
+    is_initialized: bool
+    # If none, autobackups are disabled
+    autobackup_period: typing.Optional[int]
+    # None is equal to all quotas being unlimited (-1). Optional for compatibility reasons.
+    autobackup_quotas: AutobackupQuotas
+    # Bucket name for Backblaze, path for some other providers
+    location_name: typing.Optional[str]
+    location_id: typing.Optional[str]
+
+
+@strawberry.type
+class Backup:
+    @strawberry.field
+    def configuration(self) -> BackupConfiguration:
+        return BackupConfiguration(
+            provider=Backups.provider().name,
+            encryption_key=LocalBackupSecret.get(),
+            is_initialized=Backups.is_initted(),
+            autobackup_period=Backups.autobackup_period_minutes(),
+            location_name=Backups.provider().location,
+            location_id=Backups.provider().repo_id,
+            autobackup_quotas=Backups.autobackup_quotas(),
+        )
+
+    @strawberry.field
+    def all_snapshots(self) -> typing.List[SnapshotInfo]:
+        if not Backups.is_initted():
+            return []
+        result = []
+        snapshots = Backups.get_all_snapshots()
+        for snap in snapshots:
+            service = get_service_by_id(snap.service_name)
+            if service is None:
+                service = Service(
+                    id=snap.service_name,
+                    display_name=f"{snap.service_name} (Orphaned)",
+                    description="",
+                    svg_icon="",
+                    is_movable=False,
+                    is_required=False,
+                    is_enabled=False,
+                    status=ServiceStatusEnum.OFF,
+                    url=None,
+                    dns_records=None,
+                    can_be_backed_up=False,
+                    backup_description="",
+                )
+            else:
+                service = service_to_graphql_service(service)
+            graphql_snap = SnapshotInfo(
+                id=snap.id,
+                service=service,
+                created_at=snap.created_at,
+                reason=snap.reason,
+            )
+            result.append(graphql_snap)
+        return result

selfprivacy_api/graphql/queries/jobs.py

@@ -15,10 +15,9 @@ from selfprivacy_api.jobs import Jobs
 class Job:
     @strawberry.field
     def get_jobs(self) -> typing.List[ApiJob]:
+        Jobs.get_jobs()
 
-        Jobs.get_instance().get_jobs()
-
-        return [job_to_api_job(job) for job in Jobs.get_instance().get_jobs()]
+        return [job_to_api_job(job) for job in Jobs.get_jobs()]
 
     @strawberry.field
     def get_job(self, job_id: str) -> typing.Optional[ApiJob]:

selfprivacy_api/graphql/queries/providers.py

@@ -6,8 +6,20 @@ import strawberry
 @strawberry.enum
 class DnsProvider(Enum):
     CLOUDFLARE = "CLOUDFLARE"
+    DIGITALOCEAN = "DIGITALOCEAN"
+    DESEC = "DESEC"
 
 
 @strawberry.enum
 class ServerProvider(Enum):
     HETZNER = "HETZNER"
+    DIGITALOCEAN = "DIGITALOCEAN"
+
+
+@strawberry.enum
+class BackupProvider(Enum):
+    BACKBLAZE = "BACKBLAZE"
+    NONE = "NONE"
+    # for testing purposes, make sure not selectable in prod.
+    MEMORY = "MEMORY"
+    FILE = "FILE"

selfprivacy_api/graphql/queries/storage.py

@@ -23,7 +23,7 @@ class Storage:
                 else str(volume.size),
                 free_space=str(volume.fsavail),
                 used_space=str(volume.fsused),
-                root=volume.name == "sda1",
+                root=volume.is_root(),
                 name=volume.name,
                 model=volume.model,
                 serial=volume.serial,

selfprivacy_api/graphql/queries/system.py

@@ -33,6 +33,7 @@ class SystemDomainInfo:
                 content=record.content,
                 ttl=record.ttl,
                 priority=record.priority,
+                display_name=record.display_name,
             )
             for record in get_all_required_dns_records()
         ]
@@ -44,7 +45,7 @@ def get_system_domain_info() -> SystemDomainInfo:
         return SystemDomainInfo(
             domain=user_data["domain"],
             hostname=user_data["hostname"],
-            provider=DnsProvider.CLOUDFLARE,
+            provider=user_data["dns"]["provider"],
         )
 
 
@@ -133,7 +134,11 @@ class SystemProviderInfo:
 
 def get_system_provider_info() -> SystemProviderInfo:
     """Get system provider info"""
-    return SystemProviderInfo(provider=ServerProvider.HETZNER, id="UNKNOWN")
+    with ReadUserData() as user_data:
+        return SystemProviderInfo(
+            provider=user_data["server"]["provider"],
+            id="UNKNOWN",
+        )
 
 
 @strawberry.type

selfprivacy_api/graphql/schema.py

@@ -5,21 +5,30 @@ import asyncio
 from typing import AsyncGenerator
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
+from selfprivacy_api.graphql.mutations.deprecated_mutations import (
+    DeprecatedApiMutations,
+    DeprecatedJobMutations,
+    DeprecatedServicesMutations,
+    DeprecatedStorageMutations,
+    DeprecatedSystemMutations,
+    DeprecatedUsersMutations,
+)
 from selfprivacy_api.graphql.mutations.api_mutations import ApiMutations
 from selfprivacy_api.graphql.mutations.job_mutations import JobMutations
 from selfprivacy_api.graphql.mutations.mutation_interface import GenericMutationReturn
 from selfprivacy_api.graphql.mutations.services_mutations import ServicesMutations
-from selfprivacy_api.graphql.mutations.ssh_mutations import SshMutations
 from selfprivacy_api.graphql.mutations.storage_mutations import StorageMutations
 from selfprivacy_api.graphql.mutations.system_mutations import SystemMutations
+from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
 
 from selfprivacy_api.graphql.queries.api_queries import Api
+from selfprivacy_api.graphql.queries.backup import Backup
 from selfprivacy_api.graphql.queries.jobs import Job
 from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
-from selfprivacy_api.graphql.mutations.users_mutations import UserMutations
+from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
 from selfprivacy_api.jobs.test import test_job
 
@@ -28,16 +37,16 @@ from selfprivacy_api.jobs.test import test_job
 class Query:
     """Root schema for queries"""
 
-    @strawberry.field(permission_classes=[IsAuthenticated])
-    def system(self) -> System:
-        """System queries"""
-        return System()
-
     @strawberry.field
     def api(self) -> Api:
         """API access status"""
         return Api()
 
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def system(self) -> System:
+        """System queries"""
+        return System()
+
     @strawberry.field(permission_classes=[IsAuthenticated])
     def users(self) -> Users:
         """Users queries"""
@@ -58,19 +67,58 @@ class Query:
         """Services queries"""
         return Services()
 
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def backup(self) -> Backup:
+        """Backup queries"""
+        return Backup()
+
 
 @strawberry.type
 class Mutation(
-    ApiMutations,
-    SystemMutations,
-    UserMutations,
-    SshMutations,
-    StorageMutations,
-    ServicesMutations,
-    JobMutations,
+    DeprecatedApiMutations,
+    DeprecatedSystemMutations,
+    DeprecatedUsersMutations,
+    DeprecatedStorageMutations,
+    DeprecatedServicesMutations,
+    DeprecatedJobMutations,
 ):
     """Root schema for mutations"""
 
+    @strawberry.field
+    def api(self) -> ApiMutations:
+        """API mutations"""
+        return ApiMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def system(self) -> SystemMutations:
+        """System mutations"""
+        return SystemMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def users(self) -> UsersMutations:
+        """Users mutations"""
+        return UsersMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def storage(self) -> StorageMutations:
+        """Storage mutations"""
+        return StorageMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def services(self) -> ServicesMutations:
+        """Services mutations"""
+        return ServicesMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def jobs(self) -> JobMutations:
+        """Jobs mutations"""
+        return JobMutations()
+
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def backup(self) -> BackupMutations:
+        """Backup mutations"""
+        return BackupMutations()
+
     @strawberry.mutation(permission_classes=[IsAuthenticated])
     def test_mutation(self) -> GenericMutationReturn:
         """Test mutation"""
@@ -95,4 +143,8 @@ class Subscription:
             await asyncio.sleep(0.5)
 
 
-schema = strawberry.Schema(query=Query, mutation=Mutation, subscription=Subscription)
+schema = strawberry.Schema(
+    query=Query,
+    mutation=Mutation,
+    subscription=Subscription,
+)

selfprivacy_api/jobs/__init__.py

@@ -8,8 +8,8 @@ A job is a dictionary with the following keys:
     - name: name of the job
     - description: description of the job
     - status: status of the job
-    - created_at: date of creation of the job
-    - updated_at: date of last update of the job
+    - created_at: date of creation of the job, naive localtime
+    - updated_at: date of last update of the job, naive localtime
     - finished_at: date of finish of the job
     - error: error message if the job failed
     - result: result of the job
@@ -17,19 +17,20 @@ A job is a dictionary with the following keys:
 import typing
 import datetime
 from uuid import UUID
-import asyncio
-import json
-import os
-import time
 import uuid
 from enum import Enum
 
 from pydantic import BaseModel
 
-from selfprivacy_api.utils import ReadUserData, UserDataFiles, WriteUserData
+from selfprivacy_api.utils.redis_pool import RedisPool
+
+JOB_EXPIRATION_SECONDS = 10 * 24 * 60 * 60  # ten days
+
+STATUS_LOGS_PREFIX = "jobs_logs:status:"
+PROGRESS_LOGS_PREFIX = "jobs_logs:progress:"
 
 
-class JobStatus(Enum):
+class JobStatus(str, Enum):
     """
     Status of a job.
     """
@@ -64,36 +65,15 @@ class Jobs:
     Jobs class.
     """
 
-    __instance = None
-
-    @staticmethod
-    def get_instance():
-        """
-        Singleton method.
-        """
-        if Jobs.__instance is None:
-            Jobs()
-            if Jobs.__instance is None:
-                raise Exception("Couldn't init Jobs singleton!")
-            return Jobs.__instance
-        return Jobs.__instance
-
-    def __init__(self):
-        """
-        Initialize the jobs list.
-        """
-        if Jobs.__instance is not None:
-            raise Exception("This class is a singleton!")
-        else:
-            Jobs.__instance = self
-
     @staticmethod
     def reset() -> None:
         """
         Reset the jobs list.
         """
-        with WriteUserData(UserDataFiles.JOBS) as user_data:
-            user_data["jobs"] = []
+        jobs = Jobs.get_jobs()
+        for job in jobs:
+            Jobs.remove(job)
+        Jobs.reset_logs()
 
     @staticmethod
     def add(
@@ -121,34 +101,83 @@ class Jobs:
             error=None,
             result=None,
         )
-        with WriteUserData(UserDataFiles.JOBS) as user_data:
-            try:
-                if "jobs" not in user_data:
-                    user_data["jobs"] = []
-                user_data["jobs"].append(json.loads(job.json()))
-            except json.decoder.JSONDecodeError:
-                user_data["jobs"] = [json.loads(job.json())]
+        redis = RedisPool().get_connection()
+        _store_job_as_hash(redis, _redis_key_from_uuid(job.uid), job)
         return job
 
-    def remove(self, job: Job) -> None:
+    @staticmethod
+    def remove(job: Job) -> None:
         """
         Remove a job from the jobs list.
         """
-        self.remove_by_uid(str(job.uid))
+        Jobs.remove_by_uid(str(job.uid))
 
-    def remove_by_uid(self, job_uuid: str) -> bool:
+    @staticmethod
+    def remove_by_uid(job_uuid: str) -> bool:
         """
         Remove a job from the jobs list.
         """
-        with WriteUserData(UserDataFiles.JOBS) as user_data:
-            if "jobs" not in user_data:
-                user_data["jobs"] = []
-            for i, j in enumerate(user_data["jobs"]):
-                if j["uid"] == job_uuid:
-                    del user_data["jobs"][i]
-                    return True
+        redis = RedisPool().get_connection()
+        key = _redis_key_from_uuid(job_uuid)
+        if redis.exists(key):
+            redis.delete(key)
+            return True
         return False
 
+    @staticmethod
+    def reset_logs() -> None:
+        redis = RedisPool().get_connection()
+        for key in redis.keys(STATUS_LOGS_PREFIX + "*"):
+            redis.delete(key)
+
+    @staticmethod
+    def log_status_update(job: Job, status: JobStatus) -> None:
+        redis = RedisPool().get_connection()
+        key = _status_log_key_from_uuid(job.uid)
+        redis.lpush(key, status.value)
+        redis.expire(key, 10)
+
+    @staticmethod
+    def log_progress_update(job: Job, progress: int) -> None:
+        redis = RedisPool().get_connection()
+        key = _progress_log_key_from_uuid(job.uid)
+        redis.lpush(key, progress)
+        redis.expire(key, 10)
+
+    @staticmethod
+    def status_updates(job: Job) -> list[JobStatus]:
+        result: list[JobStatus] = []
+
+        redis = RedisPool().get_connection()
+        key = _status_log_key_from_uuid(job.uid)
+        if not redis.exists(key):
+            return []
+
+        status_strings: list[str] = redis.lrange(key, 0, -1)  # type: ignore
+        for status in status_strings:
+            try:
+                result.append(JobStatus[status])
+            except KeyError as error:
+                raise ValueError("impossible job status: " + status) from error
+        return result
+
+    @staticmethod
+    def progress_updates(job: Job) -> list[int]:
+        result: list[int] = []
+
+        redis = RedisPool().get_connection()
+        key = _progress_log_key_from_uuid(job.uid)
+        if not redis.exists(key):
+            return []
+
+        progress_strings: list[str] = redis.lrange(key, 0, -1)  # type: ignore
+        for progress in progress_strings:
+            try:
+                result.append(int(progress))
+            except KeyError as error:
+                raise ValueError("impossible job progress: " + progress) from error
+        return result
+
     @staticmethod
     def update(
         job: Job,
@@ -169,36 +198,49 @@ class Jobs:
             job.description = description
         if status_text is not None:
             job.status_text = status_text
-        if progress is not None:
+
+        # if it is finished it is 100
+        # unless user says otherwise
+        if status == JobStatus.FINISHED and progress is None:
+            progress = 100
+        if progress is not None and job.progress != progress:
             job.progress = progress
+            Jobs.log_progress_update(job, progress)
+
         job.status = status
+        Jobs.log_status_update(job, status)
         job.updated_at = datetime.datetime.now()
         job.error = error
         job.result = result
         if status in (JobStatus.FINISHED, JobStatus.ERROR):
             job.finished_at = datetime.datetime.now()
 
-        with WriteUserData(UserDataFiles.JOBS) as user_data:
-            if "jobs" not in user_data:
-                user_data["jobs"] = []
-            for i, j in enumerate(user_data["jobs"]):
-                if j["uid"] == str(job.uid):
-                    user_data["jobs"][i] = json.loads(job.json())
-                    break
+        redis = RedisPool().get_connection()
+        key = _redis_key_from_uuid(job.uid)
+        if redis.exists(key):
+            _store_job_as_hash(redis, key, job)
+            if status in (JobStatus.FINISHED, JobStatus.ERROR):
+                redis.expire(key, JOB_EXPIRATION_SECONDS)
 
         return job
 
+    @staticmethod
+    def set_expiration(job: Job, expiration_seconds: int) -> Job:
+        redis = RedisPool().get_connection()
+        key = _redis_key_from_uuid(job.uid)
+        if redis.exists(key):
+            redis.expire(key, expiration_seconds)
+        return job
+
     @staticmethod
     def get_job(uid: str) -> typing.Optional[Job]:
         """
         Get a job from the jobs list.
         """
-        with ReadUserData(UserDataFiles.JOBS) as user_data:
-            if "jobs" not in user_data:
-                user_data["jobs"] = []
-            for job in user_data["jobs"]:
-                if job["uid"] == uid:
-                    return Job(**job)
+        redis = RedisPool().get_connection()
+        key = _redis_key_from_uuid(uid)
+        if redis.exists(key):
+            return _job_from_hash(redis, key)
         return None
 
     @staticmethod
@@ -206,23 +248,76 @@ class Jobs:
         """
         Get the jobs list.
         """
-        with ReadUserData(UserDataFiles.JOBS) as user_data:
-            try:
-                if "jobs" not in user_data:
-                    user_data["jobs"] = []
-                return [Job(**job) for job in user_data["jobs"]]
-            except json.decoder.JSONDecodeError:
-                return []
+        redis = RedisPool().get_connection()
+        job_keys = redis.keys("jobs:*")
+        jobs = []
+        for job_key in job_keys:
+            job = _job_from_hash(redis, job_key)
+            if job is not None:
+                jobs.append(job)
+        return jobs
 
     @staticmethod
     def is_busy() -> bool:
         """
         Check if there is a job running.
         """
-        with ReadUserData(UserDataFiles.JOBS) as user_data:
-            if "jobs" not in user_data:
-                user_data["jobs"] = []
-            for job in user_data["jobs"]:
-                if job["status"] == JobStatus.RUNNING.value:
-                    return True
+        for job in Jobs.get_jobs():
+            if job.status == JobStatus.RUNNING:
+                return True
         return False
+
+
+def report_progress(progress: int, job: Job, status_text: str) -> None:
+    """
+    A terse way to call a common operation, for readability
+    job.report_progress() would be even better
+    but it would go against how this file is written
+    """
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        status_text=status_text,
+        progress=progress,
+    )
+
+
+def _redis_key_from_uuid(uuid_string) -> str:
+    return "jobs:" + str(uuid_string)
+
+
+def _status_log_key_from_uuid(uuid_string) -> str:
+    return STATUS_LOGS_PREFIX + str(uuid_string)
+
+
+def _progress_log_key_from_uuid(uuid_string) -> str:
+    return PROGRESS_LOGS_PREFIX + str(uuid_string)
+
+
+def _store_job_as_hash(redis, redis_key, model) -> None:
+    for key, value in model.dict().items():
+        if isinstance(value, uuid.UUID):
+            value = str(value)
+        if isinstance(value, datetime.datetime):
+            value = value.isoformat()
+        if isinstance(value, JobStatus):
+            value = value.value
+        redis.hset(redis_key, key, str(value))
+
+
+def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
+    if redis.exists(redis_key):
+        job_dict = redis.hgetall(redis_key)
+        for date in [
+            "created_at",
+            "updated_at",
+            "finished_at",
+        ]:
+            if job_dict[date] != "None":
+                job_dict[date] = datetime.datetime.fromisoformat(job_dict[date])
+        for key in job_dict.keys():
+            if job_dict[key] == "None":
+                job_dict[key] = None
+
+        return Job(**job_dict)
+    return None

selfprivacy_api/jobs/migrate_to_binds.py

@@ -67,8 +67,8 @@ def move_folder(
 
     try:
         data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
-    except Exception as e:
-        print(f"Error creating data path: {e}")
+    except Exception as error:
+        print(f"Error creating data path: {error}")
         return
 
     try:

selfprivacy_api/jobs/nix_collect_garbage.py

@@ -0,0 +1,147 @@
+import re
+import subprocess
+from typing import Tuple, Iterable
+
+from selfprivacy_api.utils.huey import huey
+
+from selfprivacy_api.jobs import JobStatus, Jobs, Job
+
+
+class ShellException(Exception):
+    """Shell-related errors"""
+
+
+COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
+RESULT_WAS_NOT_FOUND_ERROR = (
+    "We are sorry, garbage collection result was not found. "
+    "Something went wrong, please report this to the support chat."
+)
+CLEAR_COMPLETED = "Garbage collection completed."
+
+
+def delete_old_gens_and_return_dead_report() -> str:
+    subprocess.run(
+        ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
+        check=False,
+    )
+
+    result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
+        "utf-8"
+    )
+
+    return " " if result is None else result
+
+
+def run_nix_collect_garbage() -> Iterable[bytes]:
+    process = subprocess.Popen(
+        ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    return process.stdout if process.stdout else iter([])
+
+
+def parse_line(job: Job, line: str) -> Job:
+    """
+    We parse the string for the presence of a final line,
+    with the final amount of space cleared.
+    Simply put, we're just looking for a similar string:
+    "1537 store paths deleted, 339.84 MiB freed".
+    """
+    pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
+    match = re.search(pattern, line)
+
+    if match is None:
+        raise ShellException("nix returned gibberish output")
+
+    else:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text=CLEAR_COMPLETED,
+            result=f"{match.group(0)} have been cleared",
+        )
+    return job
+
+
+def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
+    completed_packages = 0
+    prev_progress = 0
+
+    for line in stream:
+        line = line.decode("utf-8")
+
+        if "deleting '/nix/store/" in line:
+            completed_packages += 1
+            percent = int((completed_packages / total_dead_packages) * 100)
+
+            if percent - prev_progress >= 5:
+                Jobs.update(
+                    job=job,
+                    status=JobStatus.RUNNING,
+                    progress=percent,
+                    status_text="Cleaning...",
+                )
+                prev_progress = percent
+
+        elif "store paths deleted," in line:
+            parse_line(job, line)
+
+
+def get_dead_packages(output) -> Tuple[int, float]:
+    dead = len(re.findall("/nix/store/", output))
+    percent = 0
+    if dead != 0:
+        percent = 100 / dead
+    return dead, percent
+
+
+@huey.task()
+def calculate_and_clear_dead_paths(job: Job):
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text="Calculate the number of dead packages...",
+    )
+
+    dead_packages, package_equal_to_percent = get_dead_packages(
+        delete_old_gens_and_return_dead_report()
+    )
+
+    if dead_packages == 0:
+        Jobs.update(
+            job=job,
+            status=JobStatus.FINISHED,
+            status_text="Nothing to clear",
+            result="System is clear",
+        )
+        return True
+
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        progress=0,
+        status_text=f"Found {dead_packages} packages to remove!",
+    )
+
+    stream = run_nix_collect_garbage()
+    try:
+        process_stream(job, stream, dead_packages)
+    except ShellException as error:
+        Jobs.update(
+            job=job,
+            status=JobStatus.ERROR,
+            status_text=COMPLETED_WITH_ERROR,
+            error=RESULT_WAS_NOT_FOUND_ERROR,
+        )
+
+
+def start_nix_collect_garbage() -> Job:
+    job = Jobs.add(
+        type_id="maintenance.collect_nix_garbage",
+        name="Collect garbage",
+        description="Cleaning up unused packages",
+    )
+
+    calculate_and_clear_dead_paths(job=job)
+
+    return job

selfprivacy_api/jobs/test.py

@@ -5,7 +5,7 @@ from selfprivacy_api.jobs import JobStatus, Jobs
 
 @huey.task()
 def test_job():
-    job = Jobs.get_instance().add(
+    job = Jobs.add(
         type_id="test",
         name="Test job",
         description="This is a test job.",
@@ -14,42 +14,42 @@ def test_job():
         progress=0,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.RUNNING,
         status_text="Performing pre-move checks...",
         progress=5,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.RUNNING,
         status_text="Performing pre-move checks...",
         progress=10,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.RUNNING,
         status_text="Performing pre-move checks...",
         progress=15,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.RUNNING,
         status_text="Performing pre-move checks...",
         progress=20,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.RUNNING,
         status_text="Performing pre-move checks...",
         progress=25,
     )
     time.sleep(5)
-    Jobs.get_instance().update(
+    Jobs.update(
         job=job,
         status=JobStatus.FINISHED,
         status_text="Job finished.",

selfprivacy_api/jobs/upgrade_system.py

@@ -0,0 +1,136 @@
+"""
+A task to start the system upgrade or rebuild by starting a systemd unit.
+After starting, track the status of the systemd unit and update the Job
+status accordingly.
+"""
+import subprocess
+from selfprivacy_api.utils.huey import huey
+from selfprivacy_api.jobs import JobStatus, Jobs, Job
+from selfprivacy_api.utils.waitloop import wait_until_true
+from selfprivacy_api.utils.systemd import (
+    get_service_status,
+    get_last_log_lines,
+    ServiceStatus,
+)
+
+START_TIMEOUT = 60 * 5
+START_INTERVAL = 1
+RUN_TIMEOUT = 60 * 60
+RUN_INTERVAL = 5
+
+
+def check_if_started(unit_name: str):
+    """Check if the systemd unit has started"""
+    try:
+        status = get_service_status(unit_name)
+        if status == ServiceStatus.ACTIVE:
+            return True
+        return False
+    except subprocess.CalledProcessError:
+        return False
+
+
+def check_running_status(job: Job, unit_name: str):
+    """Check if the systemd unit is running"""
+    try:
+        status = get_service_status(unit_name)
+        if status == ServiceStatus.INACTIVE:
+            Jobs.update(
+                job=job,
+                status=JobStatus.FINISHED,
+                result="System rebuilt.",
+                progress=100,
+            )
+            return True
+        if status == ServiceStatus.FAILED:
+            log_lines = get_last_log_lines(unit_name, 10)
+            Jobs.update(
+                job=job,
+                status=JobStatus.ERROR,
+                error="System rebuild failed. Last log lines:\n" + "\n".join(log_lines),
+            )
+            return True
+        if status == ServiceStatus.ACTIVE:
+            log_lines = get_last_log_lines(unit_name, 1)
+            Jobs.update(
+                job=job,
+                status=JobStatus.RUNNING,
+                status_text=log_lines[0] if len(log_lines) > 0 else "",
+            )
+            return False
+        return False
+    except subprocess.CalledProcessError:
+        return False
+
+
+def rebuild_system(job: Job, upgrade: bool = False):
+    """
+    Broken out to allow calling it synchronously.
+    We cannot just block until task is done because it will require a second worker
+    Which we do not have
+    """
+
+    unit_name = "sp-nixos-upgrade.service" if upgrade else "sp-nixos-rebuild.service"
+    try:
+        command = ["systemctl", "start", unit_name]
+        subprocess.run(
+            command,
+            check=True,
+            start_new_session=True,
+            shell=False,
+        )
+        Jobs.update(
+            job=job,
+            status=JobStatus.RUNNING,
+            status_text="Starting the system rebuild...",
+        )
+        # Wait for the systemd unit to start
+        try:
+            wait_until_true(
+                lambda: check_if_started(unit_name),
+                timeout_sec=START_TIMEOUT,
+                interval=START_INTERVAL,
+            )
+        except TimeoutError:
+            log_lines = get_last_log_lines(unit_name, 10)
+            Jobs.update(
+                job=job,
+                status=JobStatus.ERROR,
+                error="System rebuild timed out. Last log lines:\n"
+                + "\n".join(log_lines),
+            )
+            return
+        Jobs.update(
+            job=job,
+            status=JobStatus.RUNNING,
+            status_text="Rebuilding the system...",
+        )
+        # Wait for the systemd unit to finish
+        try:
+            wait_until_true(
+                lambda: check_running_status(job, unit_name),
+                timeout_sec=RUN_TIMEOUT,
+                interval=RUN_INTERVAL,
+            )
+        except TimeoutError:
+            log_lines = get_last_log_lines(unit_name, 10)
+            Jobs.update(
+                job=job,
+                status=JobStatus.ERROR,
+                error="System rebuild timed out. Last log lines:\n"
+                + "\n".join(log_lines),
+            )
+            return
+
+    except subprocess.CalledProcessError as e:
+        Jobs.update(
+            job=job,
+            status=JobStatus.ERROR,
+            status_text=str(e),
+        )
+
+
+@huey.task()
+def rebuild_system_task(job: Job, upgrade: bool = False):
+    """Rebuild the system"""
+    rebuild_system(job, upgrade)

selfprivacy_api/migrations/__init__.py

@@ -8,21 +8,16 @@ at api.skippedMigrations in userdata.json and populating it
 with IDs of the migrations to skip.
 Adding DISABLE_ALL to that array disables the migrations module entirely.
 """
-from selfprivacy_api.migrations.check_for_failed_binds_migration import CheckForFailedBindsMigration
-from selfprivacy_api.utils import ReadUserData
-from selfprivacy_api.migrations.fix_nixos_config_branch import FixNixosConfigBranch
-from selfprivacy_api.migrations.create_tokens_json import CreateTokensJson
-from selfprivacy_api.migrations.migrate_to_selfprivacy_channel import (
-    MigrateToSelfprivacyChannel,
+
+from selfprivacy_api.utils import ReadUserData, UserDataFiles
+from selfprivacy_api.migrations.write_token_to_redis import WriteTokenToRedis
+from selfprivacy_api.migrations.check_for_system_rebuild_jobs import (
+    CheckForSystemRebuildJobs,
 )
-from selfprivacy_api.migrations.mount_volume import MountVolume
 
 migrations = [
-    FixNixosConfigBranch(),
-    CreateTokensJson(),
-    MigrateToSelfprivacyChannel(),
-    MountVolume(),
-    CheckForFailedBindsMigration(),
+    WriteTokenToRedis(),
+    CheckForSystemRebuildJobs(),
 ]
 
 
@@ -31,7 +26,7 @@ def run_migrations():
     Go over all migrations. If they are not skipped in userdata file, run them
     if the migration needed.
     """
-    with ReadUserData() as data:
+    with ReadUserData(UserDataFiles.SECRETS) as data:
         if "api" not in data:
             skipped_migrations = []
         elif "skippedMigrations" not in data["api"]:

selfprivacy_api/migrations/check_for_failed_binds_migration.py

@@ -1,48 +0,0 @@
-from selfprivacy_api.jobs import JobStatus, Jobs
-
-from selfprivacy_api.migrations.migration import Migration
-from selfprivacy_api.utils import WriteUserData
-
-
-class CheckForFailedBindsMigration(Migration):
-    """Mount volume."""
-
-    def get_migration_name(self):
-        return "check_for_failed_binds_migration"
-
-    def get_migration_description(self):
-        return "If binds migration failed, try again."
-
-    def is_migration_needed(self):
-        try:
-            jobs = Jobs.get_instance().get_jobs()
-            # If there is a job with type_id "migrations.migrate_to_binds" and status is not "FINISHED",
-            # then migration is needed and job is deleted
-            for job in jobs:
-                if (
-                    job.type_id == "migrations.migrate_to_binds"
-                    and job.status != JobStatus.FINISHED
-                ):
-                    return True
-            return False
-        except Exception as e:
-            print(e)
-            return False
-
-    def migrate(self):
-        # Get info about existing volumes
-        # Write info about volumes to userdata.json
-        try:
-            jobs = Jobs.get_instance().get_jobs()
-            for job in jobs:
-                if (
-                    job.type_id == "migrations.migrate_to_binds"
-                    and job.status != JobStatus.FINISHED
-                ):
-                    Jobs.get_instance().remove(job)
-            with WriteUserData() as userdata:
-                userdata["useBinds"] = False
-            print("Done")
-        except Exception as e:
-            print(e)
-            print("Error mounting volume")

selfprivacy_api/migrations/check_for_system_rebuild_jobs.py

@@ -0,0 +1,47 @@
+from selfprivacy_api.migrations.migration import Migration
+from selfprivacy_api.jobs import JobStatus, Jobs
+
+
+class CheckForSystemRebuildJobs(Migration):
+    """Check if there are unfinished system rebuild jobs and finish them"""
+
+    def get_migration_name(self):
+        return "check_for_system_rebuild_jobs"
+
+    def get_migration_description(self):
+        return "Check if there are unfinished system rebuild jobs and finish them"
+
+    def is_migration_needed(self):
+        # Check if there are any unfinished system rebuild jobs
+        for job in Jobs.get_jobs():
+            if (
+                job.type_id
+                in [
+                    "system.nixos.rebuild",
+                    "system.nixos.upgrade",
+                ]
+            ) and job.status in [
+                JobStatus.CREATED,
+                JobStatus.RUNNING,
+            ]:
+                return True
+
+    def migrate(self):
+        # As the API is restarted, we assume that the jobs are finished
+        for job in Jobs.get_jobs():
+            if (
+                job.type_id
+                in [
+                    "system.nixos.rebuild",
+                    "system.nixos.upgrade",
+                ]
+            ) and job.status in [
+                JobStatus.CREATED,
+                JobStatus.RUNNING,
+            ]:
+                Jobs.update(
+                    job=job,
+                    status=JobStatus.FINISHED,
+                    result="System rebuilt.",
+                    progress=100,
+                )

selfprivacy_api/migrations/create_tokens_json.py

@@ -1,58 +0,0 @@
-from datetime import datetime
-import os
-import json
-from pathlib import Path
-
-from selfprivacy_api.migrations.migration import Migration
-from selfprivacy_api.utils import TOKENS_FILE, ReadUserData
-
-
-class CreateTokensJson(Migration):
-    def get_migration_name(self):
-        return "create_tokens_json"
-
-    def get_migration_description(self):
-        return """Selfprivacy API used a single token in userdata.json for authentication.
-        This migration creates a new tokens.json file with the old token in it.
-        This migration runs if the tokens.json file does not exist.
-        Old token is located at ["api"]["token"] in userdata.json.
-        tokens.json path is declared in TOKENS_FILE imported from utils.py
-        tokens.json must have the following format:
-        {
-            "tokens": [
-                {
-                    "token": "token_string",
-                    "name": "Master Token",
-                    "date": "current date from str(datetime.now())",
-                }
-            ]
-        }
-        tokens.json must have 0600 permissions.
-        """
-
-    def is_migration_needed(self):
-        return not os.path.exists(TOKENS_FILE)
-
-    def migrate(self):
-        try:
-            print(f"Creating tokens.json file at {TOKENS_FILE}")
-            with ReadUserData() as userdata:
-                token = userdata["api"]["token"]
-            # Touch tokens.json with 0600 permissions
-            Path(TOKENS_FILE).touch(mode=0o600)
-            # Write token to tokens.json
-            structure = {
-                "tokens": [
-                    {
-                        "token": token,
-                        "name": "primary_token",
-                        "date": str(datetime.now()),
-                    }
-                ]
-            }
-            with open(TOKENS_FILE, "w", encoding="utf-8") as tokens:
-                json.dump(structure, tokens, indent=4)
-            print("Done")
-        except Exception as e:
-            print(e)
-            print("Error creating tokens.json")

selfprivacy_api/migrations/fix_nixos_config_branch.py

@@ -1,57 +0,0 @@
-import os
-import subprocess
-
-from selfprivacy_api.migrations.migration import Migration
-
-
-class FixNixosConfigBranch(Migration):
-    def get_migration_name(self):
-        return "fix_nixos_config_branch"
-
-    def get_migration_description(self):
-        return """Mobile SelfPrivacy app introduced a bug in version 0.4.0.
-        New servers were initialized with a rolling-testing nixos config branch.
-        This was fixed in app version 0.4.2, but existing servers were not updated.
-        This migration fixes this by changing the nixos config branch to master.
-        """
-
-    def is_migration_needed(self):
-        """Check the current branch of /etc/nixos and return True if it is rolling-testing"""
-        current_working_directory = os.getcwd()
-        try:
-            os.chdir("/etc/nixos")
-            nixos_config_branch = subprocess.check_output(
-                ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True
-            )
-            os.chdir(current_working_directory)
-            return nixos_config_branch.decode("utf-8").strip() == "rolling-testing"
-        except subprocess.CalledProcessError:
-            os.chdir(current_working_directory)
-            return False
-
-    def migrate(self):
-        """Affected server pulled the config with the --single-branch flag.
-        Git config remote.origin.fetch has to be changed, so all branches will be fetched.
-        Then, fetch all branches, pull and switch to master branch.
-        """
-        print("Fixing Nixos config branch")
-        current_working_directory = os.getcwd()
-        try:
-            os.chdir("/etc/nixos")
-
-            subprocess.check_output(
-                [
-                    "git",
-                    "config",
-                    "remote.origin.fetch",
-                    "+refs/heads/*:refs/remotes/origin/*",
-                ]
-            )
-            subprocess.check_output(["git", "fetch", "--all"])
-            subprocess.check_output(["git", "pull"])
-            subprocess.check_output(["git", "checkout", "master"])
-            os.chdir(current_working_directory)
-            print("Done")
-        except subprocess.CalledProcessError:
-            os.chdir(current_working_directory)
-            print("Error")

selfprivacy_api/migrations/migrate_to_selfprivacy_channel.py

@@ -1,49 +0,0 @@
-import os
-import subprocess
-
-from selfprivacy_api.migrations.migration import Migration
-
-
-class MigrateToSelfprivacyChannel(Migration):
-    """Migrate to selfprivacy Nix channel."""
-
-    def get_migration_name(self):
-        return "migrate_to_selfprivacy_channel"
-
-    def get_migration_description(self):
-        return "Migrate to selfprivacy Nix channel."
-
-    def is_migration_needed(self):
-        try:
-            output = subprocess.check_output(
-                ["nix-channel", "--list"], start_new_session=True
-            )
-            output = output.decode("utf-8")
-            first_line = output.split("\n", maxsplit=1)[0]
-            return first_line.startswith("nixos") and (
-                first_line.endswith("nixos-21.11") or first_line.endswith("nixos-21.05")
-            )
-        except subprocess.CalledProcessError:
-            return False
-
-    def migrate(self):
-        # Change the channel and update them.
-        # Also, go to /etc/nixos directory and make a git pull
-        current_working_directory = os.getcwd()
-        try:
-            print("Changing channel")
-            os.chdir("/etc/nixos")
-            subprocess.check_output(
-                [
-                    "nix-channel",
-                    "--add",
-                    "https://channel.selfprivacy.org/nixos-selfpricacy",
-                    "nixos",
-                ]
-            )
-            subprocess.check_output(["nix-channel", "--update"])
-            subprocess.check_output(["git", "pull"])
-            os.chdir(current_working_directory)
-        except subprocess.CalledProcessError:
-            os.chdir(current_working_directory)
-            print("Error")

selfprivacy_api/migrations/mount_volume.py

@@ -1,51 +0,0 @@
-import os
-import subprocess
-
-from selfprivacy_api.migrations.migration import Migration
-from selfprivacy_api.utils import ReadUserData, WriteUserData
-from selfprivacy_api.utils.block_devices import BlockDevices
-
-
-class MountVolume(Migration):
-    """Mount volume."""
-
-    def get_migration_name(self):
-        return "mount_volume"
-
-    def get_migration_description(self):
-        return "Mount volume if it is not mounted."
-
-    def is_migration_needed(self):
-        try:
-            with ReadUserData() as userdata:
-                return "volumes" not in userdata
-        except Exception as e:
-            print(e)
-            return False
-
-    def migrate(self):
-        # Get info about existing volumes
-        # Write info about volumes to userdata.json
-        try:
-            volumes = BlockDevices().get_block_devices()
-            # If there is an unmounted volume sdb,
-            # Write it to userdata.json
-            is_there_a_volume = False
-            for volume in volumes:
-                if volume.name == "sdb":
-                    is_there_a_volume = True
-                    break
-            with WriteUserData() as userdata:
-                userdata["volumes"] = []
-                if is_there_a_volume:
-                    userdata["volumes"].append(
-                        {
-                            "device": "/dev/sdb",
-                            "mountPoint": "/volumes/sdb",
-                            "fsType": "ext4",
-                        }
-                    )
-            print("Done")
-        except Exception as e:
-            print(e)
-            print("Error mounting volume")

selfprivacy_api/migrations/write_token_to_redis.py

@@ -0,0 +1,63 @@
+from datetime import datetime
+from typing import Optional
+from selfprivacy_api.migrations.migration import Migration
+from selfprivacy_api.models.tokens.token import Token
+
+from selfprivacy_api.repositories.tokens.redis_tokens_repository import (
+    RedisTokensRepository,
+)
+from selfprivacy_api.repositories.tokens.abstract_tokens_repository import (
+    AbstractTokensRepository,
+)
+from selfprivacy_api.utils import ReadUserData, UserDataFiles
+
+
+class WriteTokenToRedis(Migration):
+    """Load Json tokens into Redis"""
+
+    def get_migration_name(self):
+        return "write_token_to_redis"
+
+    def get_migration_description(self):
+        return "Loads the initial token into redis token storage"
+
+    def is_repo_empty(self, repo: AbstractTokensRepository) -> bool:
+        if repo.get_tokens() != []:
+            return False
+        return True
+
+    def get_token_from_json(self) -> Optional[Token]:
+        try:
+            with ReadUserData(UserDataFiles.SECRETS) as userdata:
+                return Token(
+                    token=userdata["api"]["token"],
+                    device_name="Initial device",
+                    created_at=datetime.now(),
+                )
+        except Exception as e:
+            print(e)
+            return None
+
+    def is_migration_needed(self):
+        try:
+            if self.get_token_from_json() is not None and self.is_repo_empty(
+                RedisTokensRepository()
+            ):
+                return True
+        except Exception as e:
+            print(e)
+            return False
+
+    def migrate(self):
+        # Write info about providers to userdata.json
+        try:
+            token = self.get_token_from_json()
+            if token is None:
+                print("No token found in secrets.json")
+                return
+            RedisTokensRepository()._store_token(token)
+
+            print("Done")
+        except Exception as e:
+            print(e)
+            print("Error migrating access tokens from json to redis")

selfprivacy_api/models/backup/provider.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+"""for storage in Redis"""
+
+
+class BackupProviderModel(BaseModel):
+    kind: str
+    login: str
+    key: str
+    location: str
+    repo_id: str  # for app usage, not for us

selfprivacy_api/models/backup/snapshot.py

@@ -0,0 +1,11 @@
+import datetime
+from pydantic import BaseModel
+
+from selfprivacy_api.graphql.common_types.backup import BackupReason
+
+
+class Snapshot(BaseModel):
+    id: str
+    service_name: str
+    created_at: datetime.datetime
+    reason: BackupReason = BackupReason.EXPLICIT

selfprivacy_api/models/services.py

@@ -0,0 +1,24 @@
+from enum import Enum
+from typing import Optional
+from pydantic import BaseModel
+
+
+class ServiceStatus(Enum):
+    """Enum for service status"""
+
+    ACTIVE = "ACTIVE"
+    RELOADING = "RELOADING"
+    INACTIVE = "INACTIVE"
+    FAILED = "FAILED"
+    ACTIVATING = "ACTIVATING"
+    DEACTIVATING = "DEACTIVATING"
+    OFF = "OFF"
+
+
+class ServiceDnsRecord(BaseModel):
+    type: str
+    name: str
+    content: str
+    ttl: int
+    display_name: str
+    priority: Optional[int] = None

selfprivacy_api/models/tokens/new_device_key.py

@@ -0,0 +1,48 @@
+"""
+New device key used to obtain access token.
+"""
+from datetime import datetime, timedelta, timezone
+import secrets
+from pydantic import BaseModel
+from mnemonic import Mnemonic
+
+from selfprivacy_api.models.tokens.time import is_past
+
+
+class NewDeviceKey(BaseModel):
+    """
+    Recovery key used to obtain access token.
+
+    Recovery key has a key string, date of creation, date of expiration.
+    """
+
+    key: str
+    created_at: datetime
+    expires_at: datetime
+
+    def is_valid(self) -> bool:
+        """
+        Check if key is valid.
+        """
+        if is_past(self.expires_at):
+            return False
+        return True
+
+    def as_mnemonic(self) -> str:
+        """
+        Get the key as a mnemonic.
+        """
+        return Mnemonic(language="english").to_mnemonic(bytes.fromhex(self.key))
+
+    @staticmethod
+    def generate() -> "NewDeviceKey":
+        """
+        Factory to generate a random token.
+        """
+        creation_date = datetime.now(timezone.utc)
+        key = secrets.token_bytes(16).hex()
+        return NewDeviceKey(
+            key=key,
+            created_at=creation_date,
+            expires_at=creation_date + timedelta(minutes=10),
+        )

selfprivacy_api/models/tokens/recovery_key.py

@@ -0,0 +1,61 @@
+"""
+Recovery key used to obtain access token.
+
+Recovery key has a token string, date of creation, optional date of expiration and optional count of uses left.
+"""
+from datetime import datetime, timezone
+import secrets
+from typing import Optional
+from pydantic import BaseModel
+from mnemonic import Mnemonic
+
+from selfprivacy_api.models.tokens.time import is_past, ensure_timezone
+
+
+class RecoveryKey(BaseModel):
+    """
+    Recovery key used to obtain access token.
+
+    Recovery key has a key string, date of creation, optional date of expiration and optional count of uses left.
+    """
+
+    key: str
+    created_at: datetime
+    expires_at: Optional[datetime]
+    uses_left: Optional[int]
+
+    def is_valid(self) -> bool:
+        """
+        Check if the recovery key is valid.
+        """
+        if self.expires_at is not None and is_past(self.expires_at):
+            return False
+        if self.uses_left is not None and self.uses_left <= 0:
+            return False
+        return True
+
+    def as_mnemonic(self) -> str:
+        """
+        Get the recovery key as a mnemonic.
+        """
+        return Mnemonic(language="english").to_mnemonic(bytes.fromhex(self.key))
+
+    @staticmethod
+    def generate(
+        expiration: Optional[datetime],
+        uses_left: Optional[int],
+    ) -> "RecoveryKey":
+        """
+        Factory to generate a random token.
+        If passed naive time as expiration, assumes utc
+        """
+        creation_date = datetime.now(timezone.utc)
+        if expiration is not None:
+            expiration = ensure_timezone(expiration)
+        key = secrets.token_bytes(24).hex()
+        return RecoveryKey(
+            key=key,
+            created_at=creation_date,
+            expires_at=expiration,
+            uses_left=uses_left,
+        )

selfprivacy_api/models/tokens/time.py

@@ -0,0 +1,14 @@
+from datetime import datetime, timezone
+
+
+def is_past(dt: datetime) -> bool:
+    # we cannot compare a naive now()
+    # to dt which might be tz-aware or unaware
+    dt = ensure_timezone(dt)
+    return dt < datetime.now(timezone.utc)
+
+
+def ensure_timezone(dt: datetime) -> datetime:
+    if dt.tzinfo is None or dt.tzinfo.utcoffset(None) is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt

selfprivacy_api/models/tokens/token.py

@@ -0,0 +1,33 @@
+"""
+Model of the access token.
+
+Access token has a token string, device name and date of creation.
+"""
+from datetime import datetime
+import secrets
+from pydantic import BaseModel
+
+
+class Token(BaseModel):
+    """
+    Model of the access token.
+
+    Access token has a token string, device name and date of creation.
+    """
+
+    token: str
+    device_name: str
+    created_at: datetime
+
+    @staticmethod
+    def generate(device_name: str) -> "Token":
+        """
+        Factory to generate a random token.
+        """
+        creation_date = datetime.now()
+        token = secrets.token_urlsafe(32)
+        return Token(
+            token=token,
+            device_name=device_name,
+            created_at=creation_date,
+        )

selfprivacy_api/repositories/tokens/abstract_tokens_repository.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Optional
+from mnemonic import Mnemonic
+from secrets import randbelow
+import re
+
+from selfprivacy_api.models.tokens.token import Token
+from selfprivacy_api.repositories.tokens.exceptions import (
+    TokenNotFound,
+    InvalidMnemonic,
+    RecoveryKeyNotFound,
+    NewDeviceKeyNotFound,
+)
+from selfprivacy_api.models.tokens.recovery_key import RecoveryKey
+from selfprivacy_api.models.tokens.new_device_key import NewDeviceKey
+
+
+class AbstractTokensRepository(ABC):
+    def get_token_by_token_string(self, token_string: str) -> Token:
+        """Get the token by token"""
+        tokens = self.get_tokens()
+        for token in tokens:
+            if token.token == token_string:
+                return token
+
+        raise TokenNotFound("Token not found!")
+
+    def get_token_by_name(self, token_name: str) -> Token:
+        """Get the token by name"""
+        tokens = self.get_tokens()
+        for token in tokens:
+            if token.device_name == token_name:
+                return token
+
+        raise TokenNotFound("Token not found!")
+
+    @abstractmethod
+    def get_tokens(self) -> list[Token]:
+        """Get the tokens"""
+
+    def create_token(self, device_name: str) -> Token:
+        """Create new token"""
+        unique_name = self._make_unique_device_name(device_name)
+        new_token = Token.generate(unique_name)
+
+        self._store_token(new_token)
+
+        return new_token
+
+    @abstractmethod
+    def delete_token(self, input_token: Token) -> None:
+        """Delete the token"""
+
+    def refresh_token(self, input_token: Token) -> Token:
+        """Change the token field of the existing token"""
+        new_token = Token.generate(device_name=input_token.device_name)
+        new_token.created_at = input_token.created_at
+
+        if input_token in self.get_tokens():
+            self.delete_token(input_token)
+            self._store_token(new_token)
+            return new_token
+
+        raise TokenNotFound("Token not found!")
+
+    def is_token_valid(self, token_string: str) -> bool:
+        """Check if the token is valid"""
+        return token_string in [token.token for token in self.get_tokens()]
+
+    def is_token_name_exists(self, token_name: str) -> bool:
+        """Check if the token name exists"""
+        return token_name in [token.device_name for token in self.get_tokens()]
+
+    def is_token_name_pair_valid(self, token_name: str, token_string: str) -> bool:
+        """Check if the token name and token are valid"""
+        try:
+            token = self.get_token_by_name(token_name)
+            if token is None:
+                return False
+        except TokenNotFound:
+            return False
+        return token.token == token_string
+
+    @abstractmethod
+    def get_recovery_key(self) -> Optional[RecoveryKey]:
+        """Get the recovery key"""
+
+    def create_recovery_key(
+        self,
+        expiration: Optional[datetime],
+        uses_left: Optional[int],
+    ) -> RecoveryKey:
+        """Create the recovery key"""
+        recovery_key = RecoveryKey.generate(expiration, uses_left)
+        self._store_recovery_key(recovery_key)
+        return recovery_key
+
+    def use_mnemonic_recovery_key(
+        self, mnemonic_phrase: str, device_name: str
+    ) -> Token:
+        """Use the mnemonic recovery key and create a new token with the given name"""
+        if not self.is_recovery_key_valid():
+            raise RecoveryKeyNotFound("Recovery key not found")
+
+        recovery_key = self.get_recovery_key()
+
+        if recovery_key is None:
+            raise RecoveryKeyNotFound("Recovery key not found")
+
+        recovery_hex_key = recovery_key.key
+        if not self._assert_mnemonic(recovery_hex_key, mnemonic_phrase):
+            raise RecoveryKeyNotFound("Recovery key not found")
+
+        new_token = self.create_token(device_name=device_name)
+
+        self._decrement_recovery_token()
+
+        return new_token
+
+    def is_recovery_key_valid(self) -> bool:
+        """Check if the recovery key is valid"""
+        recovery_key = self.get_recovery_key()
+        if recovery_key is None:
+            return False
+        return recovery_key.is_valid()
+
+    @abstractmethod
+    def _store_recovery_key(self, recovery_key: RecoveryKey) -> None:
+        """Store recovery key directly"""
+
+    @abstractmethod
+    def _delete_recovery_key(self) -> None:
+        """Delete the recovery key"""
+
+    def get_new_device_key(self) -> NewDeviceKey:
+        """Creates and returns the new device key"""
+        new_device_key = NewDeviceKey.generate()
+        self._store_new_device_key(new_device_key)
+
+        return new_device_key
+
+    def _store_new_device_key(self, new_device_key: NewDeviceKey) -> None:
+        """Store new device key directly"""
+
+    @abstractmethod
+    def delete_new_device_key(self) -> None:
+        """Delete the new device key"""
+
+    def use_mnemonic_new_device_key(
+        self, mnemonic_phrase: str, device_name: str
+    ) -> Token:
+        """Use the mnemonic new device key"""
+        new_device_key = self._get_stored_new_device_key()
+        if not new_device_key:
+            raise NewDeviceKeyNotFound
+
+        if not new_device_key.is_valid():
+            raise NewDeviceKeyNotFound
+
+        if not self._assert_mnemonic(new_device_key.key, mnemonic_phrase):
+            raise NewDeviceKeyNotFound("Phrase is not token!")
+
+        new_token = self.create_token(device_name=device_name)
+        self.delete_new_device_key()
+
+        return new_token
+
+    def reset(self):
+        for token in self.get_tokens():
+            self.delete_token(token)
+        self.delete_new_device_key()
+        self._delete_recovery_key()
+
+    def clone(self, source: AbstractTokensRepository) -> None:
+        """Clone the state of another repository to this one"""
+        self.reset()
+        for token in source.get_tokens():
+            self._store_token(token)
+
+        recovery_key = source.get_recovery_key()
+        if recovery_key is not None:
+            self._store_recovery_key(recovery_key)
+
+        new_device_key = source._get_stored_new_device_key()
+        if new_device_key is not None:
+            self._store_new_device_key(new_device_key)
+
+    @abstractmethod
+    def _store_token(self, new_token: Token):
+        """Store a token directly"""
+
+    @abstractmethod
+    def _decrement_recovery_token(self):
+        """Decrement recovery key use count by one"""
+
+    @abstractmethod
+    def _get_stored_new_device_key(self) -> Optional[NewDeviceKey]:
+        """Retrieves new device key that is already stored."""
+
+    def _make_unique_device_name(self, name: str) -> str:
+        """Token name must be an alphanumeric string and not empty.
+        Replace invalid characters with '_'
+        If name exists, add a random number to the end of the name until it is unique.
+        """
+        if not re.match("^[a-zA-Z0-9]*$", name):
+            name = re.sub("[^a-zA-Z0-9]", "_", name)
+        if name == "":
+            name = "Unknown device"
+        while self.is_token_name_exists(name):
+            name += str(randbelow(10))
+        return name
+
+    # TODO: find a proper place for it
+    def _assert_mnemonic(self, hex_key: str, mnemonic_phrase: str):
+        """Return true if hex string matches the phrase, false otherwise
+        Raise an InvalidMnemonic error if not mnemonic"""
+        recovery_token = bytes.fromhex(hex_key)
+        if not Mnemonic(language="english").check(mnemonic_phrase):
+            raise InvalidMnemonic("Phrase is not mnemonic!")
+
+        phrase_bytes = Mnemonic(language="english").to_entropy(mnemonic_phrase)
+        return phrase_bytes == recovery_token

selfprivacy_api/repositories/tokens/exceptions.py

@@ -0,0 +1,14 @@
+class TokenNotFound(Exception):
+    """Token not found!"""
+
+
+class RecoveryKeyNotFound(Exception):
+    """Recovery key not found!"""
+
+
+class InvalidMnemonic(Exception):
+    """Phrase is not mnemonic!"""
+
+
+class NewDeviceKeyNotFound(Exception):
+    """New device key not found!"""

selfprivacy_api/repositories/tokens/redis_tokens_repository.py

@@ -0,0 +1,168 @@
+"""
+Token repository using Redis as backend.
+"""
+from typing import Any, Optional
+from datetime import datetime
+from hashlib import md5
+from datetime import timezone
+
+from selfprivacy_api.repositories.tokens.abstract_tokens_repository import (
+    AbstractTokensRepository,
+)
+from selfprivacy_api.utils.redis_pool import RedisPool
+from selfprivacy_api.models.tokens.token import Token
+from selfprivacy_api.models.tokens.recovery_key import RecoveryKey
+from selfprivacy_api.models.tokens.new_device_key import NewDeviceKey
+from selfprivacy_api.repositories.tokens.exceptions import TokenNotFound
+
+TOKENS_PREFIX = "token_repo:tokens:"
+NEW_DEVICE_KEY_REDIS_KEY = "token_repo:new_device_key"
+RECOVERY_KEY_REDIS_KEY = "token_repo:recovery_key"
+
+
+class RedisTokensRepository(AbstractTokensRepository):
+    """
+    Token repository using Redis as a backend
+    """
+
+    def __init__(self):
+        self.connection = RedisPool().get_connection()
+
+    @staticmethod
+    def token_key_for_device(device_name: str):
+        md5_hash = md5(usedforsecurity=False)
+        md5_hash.update(bytes(device_name, "utf-8"))
+        digest = md5_hash.hexdigest()
+        return TOKENS_PREFIX + digest
+
+    def get_tokens(self) -> list[Token]:
+        """Get the tokens"""
+        redis = self.connection
+        token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*")  # type: ignore
+        tokens = []
+        for key in token_keys:
+            token = self._token_from_hash(key)
+            if token is not None:
+                tokens.append(token)
+        return tokens
+
+    def _discover_token_key(self, input_token: Token) -> Optional[str]:
+        """brute-force searching for tokens, for robust deletion"""
+        redis = self.connection
+        token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*")  # type: ignore
+        for key in token_keys:
+            token = self._token_from_hash(key)
+            if token == input_token:
+                return key
+        return None
+
+    def delete_token(self, input_token: Token) -> None:
+        """Delete the token"""
+        redis = self.connection
+        key = self._discover_token_key(input_token)
+        if key is None:
+            raise TokenNotFound
+        redis.delete(key)
+
+    def get_recovery_key(self) -> Optional[RecoveryKey]:
+        """Get the recovery key"""
+        redis = self.connection
+        if redis.exists(RECOVERY_KEY_REDIS_KEY):
+            return self._recovery_key_from_hash(RECOVERY_KEY_REDIS_KEY)
+        return None
+
+    def _store_recovery_key(self, recovery_key: RecoveryKey) -> None:
+        self._store_model_as_hash(RECOVERY_KEY_REDIS_KEY, recovery_key)
+
+    def _delete_recovery_key(self) -> None:
+        """Delete the recovery key"""
+        redis = self.connection
+        redis.delete(RECOVERY_KEY_REDIS_KEY)
+
+    def _store_new_device_key(self, new_device_key: NewDeviceKey) -> None:
+        """Store new device key directly"""
+        self._store_model_as_hash(NEW_DEVICE_KEY_REDIS_KEY, new_device_key)
+
+    def delete_new_device_key(self) -> None:
+        """Delete the new device key"""
+        redis = self.connection
+        redis.delete(NEW_DEVICE_KEY_REDIS_KEY)
+
+    @staticmethod
+    def _token_redis_key(token: Token) -> str:
+        return RedisTokensRepository.token_key_for_device(token.device_name)
+
+    def _store_token(self, new_token: Token):
+        """Store a token directly"""
+        key = RedisTokensRepository._token_redis_key(new_token)
+        self._store_model_as_hash(key, new_token)
+
+    def _decrement_recovery_token(self):
+        """Decrement recovery key use count by one"""
+        if self.is_recovery_key_valid():
+            recovery_key = self.get_recovery_key()
+            if recovery_key is None:
+                return
+            uses_left = recovery_key.uses_left
+            if uses_left is not None:
+                redis = self.connection
+                redis.hset(RECOVERY_KEY_REDIS_KEY, "uses_left", uses_left - 1)
+
+    def _get_stored_new_device_key(self) -> Optional[NewDeviceKey]:
+        """Retrieves new device key that is already stored."""
+        return self._new_device_key_from_hash(NEW_DEVICE_KEY_REDIS_KEY)
+
+    @staticmethod
+    def _is_date_key(key: str) -> bool:
+        return key in [
+            "created_at",
+            "expires_at",
+        ]
+
+    @staticmethod
+    def _prepare_model_dict(model_dict: dict[str, Any]) -> None:
+        date_keys = [
+            key for key in model_dict.keys() if RedisTokensRepository._is_date_key(key)
+        ]
+        for date in date_keys:
+            if model_dict[date] != "None":
+                model_dict[date] = datetime.fromisoformat(model_dict[date])
+        for key in model_dict.keys():
+            if model_dict[key] == "None":
+                model_dict[key] = None
+
+    def _model_dict_from_hash(self, redis_key: str) -> Optional[dict[str, Any]]:
+        redis = self.connection
+        if redis.exists(redis_key):
+            token_dict: dict[str, Any] = redis.hgetall(redis_key)  # type: ignore
+            RedisTokensRepository._prepare_model_dict(token_dict)
+            return token_dict
+        return None
+
+    def _hash_as_model(self, redis_key: str, model_class):
+        token_dict = self._model_dict_from_hash(redis_key)
+        if token_dict is not None:
+            return model_class(**token_dict)
+        return None
+
+    def _token_from_hash(self, redis_key: str) -> Optional[Token]:
+        token = self._hash_as_model(redis_key, Token)
+        if token is not None:
+            token.created_at = token.created_at.replace(tzinfo=None)
+            return token
+        return None
+
+    def _recovery_key_from_hash(self, redis_key: str) -> Optional[RecoveryKey]:
+        return self._hash_as_model(redis_key, RecoveryKey)
+
+    def _new_device_key_from_hash(self, redis_key: str) -> Optional[NewDeviceKey]:
+        return self._hash_as_model(redis_key, NewDeviceKey)
+
+    def _store_model_as_hash(self, redis_key, model):
+        redis = self.connection
+        for key, value in model.dict().items():
+            if isinstance(value, datetime):
+                if value.tzinfo is None:
+                    value = value.replace(tzinfo=timezone.utc)
+                value = value.isoformat()
+            redis.hset(redis_key, key, str(value))

selfprivacy_api/rest/api_auth.py

@@ -1,127 +0,0 @@
-from datetime import datetime
-from typing import Optional
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from selfprivacy_api.actions.api_tokens import (
-    CannotDeleteCallerException,
-    InvalidExpirationDate,
-    InvalidUsesLeft,
-    NotFoundException,
-    delete_api_token,
-    get_api_recovery_token_status,
-    get_api_tokens_with_caller_flag,
-    get_new_api_recovery_key,
-    refresh_api_token,
-)
-
-from selfprivacy_api.dependencies import TokenHeader, get_token_header
-
-from selfprivacy_api.utils.auth import (
-    delete_new_device_auth_token,
-    get_new_device_auth_token,
-    use_mnemonic_recoverery_token,
-    use_new_device_auth_token,
-)
-
-router = APIRouter(
-    prefix="/auth",
-    tags=["auth"],
-    responses={404: {"description": "Not found"}},
-)
-
-
-@router.get("/tokens")
-async def rest_get_tokens(auth_token: TokenHeader = Depends(get_token_header)):
-    """Get the tokens info"""
-    return get_api_tokens_with_caller_flag(auth_token.token)
-
-
-class DeleteTokenInput(BaseModel):
-    """Delete token input"""
-
-    token_name: str
-
-
-@router.delete("/tokens")
-async def rest_delete_tokens(
-    token: DeleteTokenInput, auth_token: TokenHeader = Depends(get_token_header)
-):
-    """Delete the tokens"""
-    try:
-        delete_api_token(auth_token.token, token.token_name)
-    except NotFoundException:
-        raise HTTPException(status_code=404, detail="Token not found")
-    except CannotDeleteCallerException:
-        raise HTTPException(status_code=400, detail="Cannot delete caller's token")
-    return {"message": "Token deleted"}
-
-
-@router.post("/tokens")
-async def rest_refresh_token(auth_token: TokenHeader = Depends(get_token_header)):
-    """Refresh the token"""
-    try:
-        new_token = refresh_api_token(auth_token.token)
-    except NotFoundException:
-        raise HTTPException(status_code=404, detail="Token not found")
-    return {"token": new_token}
-
-
-@router.get("/recovery_token")
-async def rest_get_recovery_token_status(
-    auth_token: TokenHeader = Depends(get_token_header),
-):
-    return get_api_recovery_token_status()
-
-
-class CreateRecoveryTokenInput(BaseModel):
-    expiration: Optional[datetime] = None
-    uses: Optional[int] = None
-
-
-@router.post("/recovery_token")
-async def rest_create_recovery_token(
-    limits: CreateRecoveryTokenInput = CreateRecoveryTokenInput(),
-    auth_token: TokenHeader = Depends(get_token_header),
-):
-    try:
-        token = get_new_api_recovery_key(limits.expiration, limits.uses)
-    except InvalidExpirationDate as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except InvalidUsesLeft as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    return {"token": token}
-
-
-class UseTokenInput(BaseModel):
-    token: str
-    device: str
-
-
-@router.post("/recovery_token/use")
-async def rest_use_recovery_token(input: UseTokenInput):
-    token = use_mnemonic_recoverery_token(input.token, input.device)
-    if token is None:
-        raise HTTPException(status_code=404, detail="Token not found")
-    return {"token": token}
-
-
-@router.post("/new_device")
-async def rest_new_device(auth_token: TokenHeader = Depends(get_token_header)):
-    token = get_new_device_auth_token()
-    return {"token": token}
-
-
-@router.delete("/new_device")
-async def rest_delete_new_device_token(
-    auth_token: TokenHeader = Depends(get_token_header),
-):
-    delete_new_device_auth_token()
-    return {"token": None}
-
-
-@router.post("/new_device/authorize")
-async def rest_new_device_authorize(input: UseTokenInput):
-    token = use_new_device_auth_token(input.token, input.device)
-    if token is None:
-        raise HTTPException(status_code=404, detail="Token not found")
-    return {"message": "Device authorized", "token": token}

selfprivacy_api/rest/services.py

@@ -1,373 +0,0 @@
-"""Basic services legacy api"""
-import base64
-from typing import Optional
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from selfprivacy_api.actions.ssh import (
-    InvalidPublicKey,
-    KeyAlreadyExists,
-    KeyNotFound,
-    create_ssh_key,
-    enable_ssh,
-    get_ssh_settings,
-    remove_ssh_key,
-    set_ssh_settings,
-)
-from selfprivacy_api.actions.users import UserNotFound, get_user_by_username
-
-from selfprivacy_api.dependencies import get_token_header
-from selfprivacy_api.restic_controller import ResticController, ResticStates
-from selfprivacy_api.restic_controller import tasks as restic_tasks
-from selfprivacy_api.services.bitwarden import Bitwarden
-from selfprivacy_api.services.gitea import Gitea
-from selfprivacy_api.services.mailserver import MailServer
-from selfprivacy_api.services.nextcloud import Nextcloud
-from selfprivacy_api.services.ocserv import Ocserv
-from selfprivacy_api.services.pleroma import Pleroma
-from selfprivacy_api.services.service import ServiceStatus
-from selfprivacy_api.utils import WriteUserData, get_dkim_key, get_domain
-
-router = APIRouter(
-    prefix="/services",
-    tags=["services"],
-    dependencies=[Depends(get_token_header)],
-    responses={404: {"description": "Not found"}},
-)
-
-
-def service_status_to_return_code(status: ServiceStatus):
-    """Converts service status object to return code for
-    compatibility with legacy api"""
-    if status == ServiceStatus.ACTIVE:
-        return 0
-    elif status == ServiceStatus.FAILED:
-        return 1
-    elif status == ServiceStatus.INACTIVE:
-        return 3
-    elif status == ServiceStatus.OFF:
-        return 4
-    else:
-        return 2
-
-
-@router.get("/status")
-async def get_status():
-    """Get the status of the services"""
-    mail_status = MailServer.get_status()
-    bitwarden_status = Bitwarden.get_status()
-    gitea_status = Gitea.get_status()
-    nextcloud_status = Nextcloud.get_status()
-    ocserv_stauts = Ocserv.get_status()
-    pleroma_status = Pleroma.get_status()
-
-    return {
-        "imap": service_status_to_return_code(mail_status),
-        "smtp": service_status_to_return_code(mail_status),
-        "http": 0,
-        "bitwarden": service_status_to_return_code(bitwarden_status),
-        "gitea": service_status_to_return_code(gitea_status),
-        "nextcloud": service_status_to_return_code(nextcloud_status),
-        "ocserv": service_status_to_return_code(ocserv_stauts),
-        "pleroma": service_status_to_return_code(pleroma_status),
-    }
-
-
-@router.post("/bitwarden/enable")
-async def enable_bitwarden():
-    """Enable Bitwarden"""
-    Bitwarden.enable()
-    return {
-        "status": 0,
-        "message": "Bitwarden enabled",
-    }
-
-
-@router.post("/bitwarden/disable")
-async def disable_bitwarden():
-    """Disable Bitwarden"""
-    Bitwarden.disable()
-    return {
-        "status": 0,
-        "message": "Bitwarden disabled",
-    }
-
-
-@router.post("/gitea/enable")
-async def enable_gitea():
-    """Enable Gitea"""
-    Gitea.enable()
-    return {
-        "status": 0,
-        "message": "Gitea enabled",
-    }
-
-
-@router.post("/gitea/disable")
-async def disable_gitea():
-    """Disable Gitea"""
-    Gitea.disable()
-    return {
-        "status": 0,
-        "message": "Gitea disabled",
-    }
-
-
-@router.get("/mailserver/dkim")
-async def get_mailserver_dkim():
-    """Get the DKIM record for the mailserver"""
-    domain = get_domain()
-
-    dkim = get_dkim_key(domain)
-    if dkim is None:
-        raise HTTPException(status_code=404, detail="DKIM record not found")
-    dkim = base64.b64encode(dkim.encode("utf-8")).decode("utf-8")
-    return dkim
-
-
-@router.post("/nextcloud/enable")
-async def enable_nextcloud():
-    """Enable Nextcloud"""
-    Nextcloud.enable()
-    return {
-        "status": 0,
-        "message": "Nextcloud enabled",
-    }
-
-
-@router.post("/nextcloud/disable")
-async def disable_nextcloud():
-    """Disable Nextcloud"""
-    Nextcloud.disable()
-    return {
-        "status": 0,
-        "message": "Nextcloud disabled",
-    }
-
-
-@router.post("/ocserv/enable")
-async def enable_ocserv():
-    """Enable Ocserv"""
-    Ocserv.enable()
-    return {
-        "status": 0,
-        "message": "Ocserv enabled",
-    }
-
-
-@router.post("/ocserv/disable")
-async def disable_ocserv():
-    """Disable Ocserv"""
-    Ocserv.disable()
-    return {
-        "status": 0,
-        "message": "Ocserv disabled",
-    }
-
-
-@router.post("/pleroma/enable")
-async def enable_pleroma():
-    """Enable Pleroma"""
-    Pleroma.enable()
-    return {
-        "status": 0,
-        "message": "Pleroma enabled",
-    }
-
-
-@router.post("/pleroma/disable")
-async def disable_pleroma():
-    """Disable Pleroma"""
-    Pleroma.disable()
-    return {
-        "status": 0,
-        "message": "Pleroma disabled",
-    }
-
-
-@router.get("/restic/backup/list")
-async def get_restic_backup_list():
-    restic = ResticController()
-    return restic.snapshot_list
-
-
-@router.put("/restic/backup/create")
-async def create_restic_backup():
-    restic = ResticController()
-    if restic.state is ResticStates.NO_KEY:
-        raise HTTPException(status_code=400, detail="Backup key not provided")
-    if restic.state is ResticStates.INITIALIZING:
-        raise HTTPException(status_code=400, detail="Backup is initializing")
-    if restic.state is ResticStates.BACKING_UP:
-        raise HTTPException(status_code=409, detail="Backup is already running")
-    restic_tasks.start_backup()
-    return {
-        "status": 0,
-        "message": "Backup creation has started",
-    }
-
-
-@router.get("/restic/backup/status")
-async def get_restic_backup_status():
-    restic = ResticController()
-
-    return {
-        "status": restic.state.name,
-        "progress": restic.progress,
-        "error_message": restic.error_message,
-    }
-
-
-@router.get("/restic/backup/reload")
-async def reload_restic_backup():
-    restic_tasks.load_snapshots()
-    return {
-        "status": 0,
-        "message": "Snapshots reload started",
-    }
-
-
-class BackupRestoreInput(BaseModel):
-    backupId: str
-
-
-@router.put("/restic/backup/restore")
-async def restore_restic_backup(backup: BackupRestoreInput):
-    restic = ResticController()
-    if restic.state is ResticStates.NO_KEY:
-        raise HTTPException(status_code=400, detail="Backup key not provided")
-    if restic.state is ResticStates.NOT_INITIALIZED:
-        raise HTTPException(
-            status_code=400, detail="Backups repository is not initialized"
-        )
-    if restic.state is ResticStates.BACKING_UP:
-        raise HTTPException(status_code=409, detail="Backup is already running")
-    if restic.state is ResticStates.INITIALIZING:
-        raise HTTPException(status_code=400, detail="Repository is initializing")
-    if restic.state is ResticStates.RESTORING:
-        raise HTTPException(status_code=409, detail="Restore is already running")
-
-    for backup_item in restic.snapshot_list:
-        if backup_item["short_id"] == backup.backupId:
-            restic_tasks.restore_from_backup(backup.backupId)
-            return {
-                "status": 0,
-                "message": "Backup restoration procedure started",
-            }
-
-    raise HTTPException(status_code=404, detail="Backup not found")
-
-
-class BackblazeConfigInput(BaseModel):
-    accountId: str
-    accountKey: str
-    bucket: str
-
-
-@router.put("/restic/backblaze/config")
-async def set_backblaze_config(backblaze_config: BackblazeConfigInput):
-    with WriteUserData() as data:
-        if "backblaze" not in data:
-            data["backblaze"] = {}
-        data["backblaze"]["accountId"] = backblaze_config.accountId
-        data["backblaze"]["accountKey"] = backblaze_config.accountKey
-        data["backblaze"]["bucket"] = backblaze_config.bucket
-
-    restic_tasks.update_keys_from_userdata()
-
-    return "New Backblaze settings saved"
-
-
-@router.post("/ssh/enable")
-async def rest_enable_ssh():
-    """Enable SSH"""
-    enable_ssh()
-    return {
-        "status": 0,
-        "message": "SSH enabled",
-    }
-
-
-@router.get("/ssh")
-async def rest_get_ssh():
-    """Get the SSH configuration"""
-    settings = get_ssh_settings()
-    return {
-        "enable": settings.enable,
-        "passwordAuthentication": settings.passwordAuthentication,
-    }
-
-
-class SshConfigInput(BaseModel):
-    enable: Optional[bool] = None
-    passwordAuthentication: Optional[bool] = None
-
-
-@router.put("/ssh")
-async def rest_set_ssh(ssh_config: SshConfigInput):
-    """Set the SSH configuration"""
-    set_ssh_settings(ssh_config.enable, ssh_config.passwordAuthentication)
-
-    return "SSH settings changed"
-
-
-class SshKeyInput(BaseModel):
-    public_key: str
-
-
-@router.put("/ssh/key/send", status_code=201)
-async def rest_send_ssh_key(input: SshKeyInput):
-    """Send the SSH key"""
-    try:
-        create_ssh_key("root", input.public_key)
-    except KeyAlreadyExists as error:
-        raise HTTPException(status_code=409, detail="Key already exists") from error
-    except InvalidPublicKey as error:
-        raise HTTPException(
-            status_code=400,
-            detail="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
-        ) from error
-
-    return {
-        "status": 0,
-        "message": "SSH key sent",
-    }
-
-
-@router.get("/ssh/keys/{username}")
-async def rest_get_ssh_keys(username: str):
-    """Get the SSH keys for a user"""
-    user = get_user_by_username(username)
-    if user is None:
-        raise HTTPException(status_code=404, detail="User not found")
-
-    return user.ssh_keys
-
-
-@router.post("/ssh/keys/{username}", status_code=201)
-async def rest_add_ssh_key(username: str, input: SshKeyInput):
-    try:
-        create_ssh_key(username, input.public_key)
-    except KeyAlreadyExists as error:
-        raise HTTPException(status_code=409, detail="Key already exists") from error
-    except InvalidPublicKey as error:
-        raise HTTPException(
-            status_code=400,
-            detail="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
-        ) from error
-    except UserNotFound as error:
-        raise HTTPException(status_code=404, detail="User not found") from error
-
-    return {
-        "message": "New SSH key successfully written",
-    }
-
-
-@router.delete("/ssh/keys/{username}")
-async def rest_delete_ssh_key(username: str, input: SshKeyInput):
-    try:
-        remove_ssh_key(username, input.public_key)
-    except KeyNotFound as error:
-        raise HTTPException(status_code=404, detail="Key not found") from error
-    except UserNotFound as error:
-        raise HTTPException(status_code=404, detail="User not found") from error
-    return {"message": "SSH key deleted"}

selfprivacy_api/rest/system.py

@@ -1,105 +0,0 @@
-from typing import Optional
-from fastapi import APIRouter, Body, Depends, HTTPException
-from pydantic import BaseModel
-
-from selfprivacy_api.dependencies import get_token_header
-
-import selfprivacy_api.actions.system as system_actions
-
-router = APIRouter(
-    prefix="/system",
-    tags=["system"],
-    dependencies=[Depends(get_token_header)],
-    responses={404: {"description": "Not found"}},
-)
-
-
-@router.get("/configuration/timezone")
-async def get_timezone():
-    """Get the timezone of the server"""
-    return system_actions.get_timezone()
-
-
-class ChangeTimezoneRequestBody(BaseModel):
-    """Change the timezone of the server"""
-
-    timezone: str
-
-
-@router.put("/configuration/timezone")
-async def change_timezone(timezone: ChangeTimezoneRequestBody):
-    """Change the timezone of the server"""
-    try:
-        system_actions.change_timezone(timezone.timezone)
-    except system_actions.InvalidTimezone as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    return {"timezone": timezone.timezone}
-
-
-@router.get("/configuration/autoUpgrade")
-async def get_auto_upgrade_settings():
-    """Get the auto-upgrade settings"""
-    return system_actions.get_auto_upgrade_settings().dict()
-
-
-class AutoUpgradeSettings(BaseModel):
-    """Settings for auto-upgrading user data"""
-
-    enable: Optional[bool] = None
-    allowReboot: Optional[bool] = None
-
-
-@router.put("/configuration/autoUpgrade")
-async def set_auto_upgrade_settings(settings: AutoUpgradeSettings):
-    """Set the auto-upgrade settings"""
-    system_actions.set_auto_upgrade_settings(settings.enable, settings.allowReboot)
-    return "Auto-upgrade settings changed"
-
-
-@router.get("/configuration/apply")
-async def apply_configuration():
-    """Apply the configuration"""
-    return_code = system_actions.rebuild_system()
-    return return_code
-
-
-@router.get("/configuration/rollback")
-async def rollback_configuration():
-    """Rollback the configuration"""
-    return_code = system_actions.rollback_system()
-    return return_code
-
-
-@router.get("/configuration/upgrade")
-async def upgrade_configuration():
-    """Upgrade the configuration"""
-    return_code = system_actions.upgrade_system()
-    return return_code
-
-
-@router.get("/reboot")
-async def reboot_system():
-    """Reboot the system"""
-    system_actions.reboot_system()
-    return "System reboot has started"
-
-
-@router.get("/version")
-async def get_system_version():
-    """Get the system version"""
-    return {"system_version": system_actions.get_system_version()}
-
-
-@router.get("/pythonVersion")
-async def get_python_version():
-    """Get the Python version"""
-    return system_actions.get_python_version()
-
-
-@router.get("/configuration/pull")
-async def pull_configuration():
-    """Pull the configuration"""
-    action_result = system_actions.pull_repository_changes()
-    if action_result.status == 0:
-        return action_result.dict()
-    raise HTTPException(status_code=500, detail=action_result.dict())

selfprivacy_api/rest/users.py

@@ -1,62 +0,0 @@
-"""Users management module"""
-from typing import Optional
-from fastapi import APIRouter, Body, Depends, HTTPException
-from pydantic import BaseModel
-
-import selfprivacy_api.actions.users as users_actions
-
-from selfprivacy_api.dependencies import get_token_header
-
-router = APIRouter(
-    prefix="/users",
-    tags=["users"],
-    dependencies=[Depends(get_token_header)],
-    responses={404: {"description": "Not found"}},
-)
-
-
-@router.get("")
-async def get_users(withMainUser: bool = False):
-    """Get the list of users"""
-    users: list[users_actions.UserDataUser] = users_actions.get_users(
-        exclude_primary=not withMainUser, exclude_root=True
-    )
-
-    return [user.username for user in users]
-
-
-class UserInput(BaseModel):
-    """User input"""
-
-    username: str
-    password: str
-
-
-@router.post("", status_code=201)
-async def create_user(user: UserInput):
-    try:
-        users_actions.create_user(user.username, user.password)
-    except users_actions.PasswordIsEmpty as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except users_actions.UsernameForbidden as e:
-        raise HTTPException(status_code=409, detail=str(e))
-    except users_actions.UsernameNotAlphanumeric as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except users_actions.UsernameTooLong as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except users_actions.UserAlreadyExists as e:
-        raise HTTPException(status_code=409, detail=str(e))
-
-    return {"result": 0, "username": user.username}
-
-
-@router.delete("/{username}")
-async def delete_user(username: str):
-    try:
-        users_actions.delete_user(username)
-    except users_actions.UserNotFound as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    except users_actions.UserIsProtected as e:
-        raise HTTPException(status_code=400, detail=str(e))
-
-    return {"result": 0, "username": username}

selfprivacy_api/restic_controller/__init__.py

@@ -1,251 +0,0 @@
-"""Restic singleton controller."""
-from datetime import datetime
-import json
-import subprocess
-import os
-from threading import Lock
-from enum import Enum
-import portalocker
-from selfprivacy_api.utils import ReadUserData
-
-
-class ResticStates(Enum):
-    """Restic states enum."""
-
-    NO_KEY = 0
-    NOT_INITIALIZED = 1
-    INITIALIZED = 2
-    BACKING_UP = 3
-    RESTORING = 4
-    ERROR = 5
-    INITIALIZING = 6
-
-
-class ResticController:
-    """
-    States in wich the restic_controller may be
-    - no backblaze key
-    - backblaze key is provided, but repository is not initialized
-    - backblaze key is provided, repository is initialized
-    - fetching list of snapshots
-    - creating snapshot, current progress can be retrieved
-    - recovering from snapshot
-
-    Any ongoing operation acquires the lock
-    Current state can be fetched with get_state()
-    """
-
-    _instance = None
-    _lock = Lock()
-    _initialized = False
-
-    def __new__(cls):
-        if not cls._instance:
-            with cls._lock:
-                cls._instance = super(ResticController, cls).__new__(cls)
-        return cls._instance
-
-    def __init__(self):
-        if self._initialized:
-            return
-        self.state = ResticStates.NO_KEY
-        self.lock = False
-        self.progress = 0
-        self._backblaze_account = None
-        self._backblaze_key = None
-        self._repository_name = None
-        self.snapshot_list = []
-        self.error_message = None
-        self._initialized = True
-        self.load_configuration()
-        self.write_rclone_config()
-        self.load_snapshots()
-
-    def load_configuration(self):
-        """Load current configuration from user data to singleton."""
-        with ReadUserData() as user_data:
-            self._backblaze_account = user_data["backblaze"]["accountId"]
-            self._backblaze_key = user_data["backblaze"]["accountKey"]
-            self._repository_name = user_data["backblaze"]["bucket"]
-        if self._backblaze_account and self._backblaze_key and self._repository_name:
-            self.state = ResticStates.INITIALIZING
-        else:
-            self.state = ResticStates.NO_KEY
-
-    def write_rclone_config(self):
-        """
-        Open /root/.config/rclone/rclone.conf with portalocker
-        and write configuration in the following format:
-            [backblaze]
-            type = b2
-            account = {self.backblaze_account}
-            key = {self.backblaze_key}
-        """
-        with portalocker.Lock(
-            "/root/.config/rclone/rclone.conf", "w", timeout=None
-        ) as rclone_config:
-            rclone_config.write(
-                f"[backblaze]\n"
-                f"type = b2\n"
-                f"account = {self._backblaze_account}\n"
-                f"key = {self._backblaze_key}\n"
-            )
-
-    def load_snapshots(self):
-        """
-        Load list of snapshots from repository
-        """
-        backup_listing_command = [
-            "restic",
-            "-o",
-            "rclone.args=serve restic --stdio",
-            "-r",
-            f"rclone:backblaze:{self._repository_name}/sfbackup",
-            "snapshots",
-            "--json",
-        ]
-
-        if self.state in (ResticStates.BACKING_UP, ResticStates.RESTORING):
-            return
-        with subprocess.Popen(
-            backup_listing_command,
-            shell=False,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-        ) as backup_listing_process_descriptor:
-            snapshots_list = backup_listing_process_descriptor.communicate()[0].decode(
-                "utf-8"
-            )
-        try:
-            starting_index = snapshots_list.find("[")
-            json.loads(snapshots_list[starting_index:])
-            self.snapshot_list = json.loads(snapshots_list[starting_index:])
-            self.state = ResticStates.INITIALIZED
-            print(snapshots_list)
-        except ValueError:
-            if "Is there a repository at the following location?" in snapshots_list:
-                self.state = ResticStates.NOT_INITIALIZED
-                return
-            self.state = ResticStates.ERROR
-            self.error_message = snapshots_list
-            return
-
-    def initialize_repository(self):
-        """
-        Initialize repository with restic
-        """
-        initialize_repository_command = [
-            "restic",
-            "-o",
-            "rclone.args=serve restic --stdio",
-            "-r",
-            f"rclone:backblaze:{self._repository_name}/sfbackup",
-            "init",
-        ]
-        with subprocess.Popen(
-            initialize_repository_command,
-            shell=False,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-        ) as initialize_repository_process_descriptor:
-            msg = initialize_repository_process_descriptor.communicate()[0].decode(
-                "utf-8"
-            )
-            if initialize_repository_process_descriptor.returncode == 0:
-                self.state = ResticStates.INITIALIZED
-            else:
-                self.state = ResticStates.ERROR
-                self.error_message = msg
-
-        self.state = ResticStates.INITIALIZED
-
-    def start_backup(self):
-        """
-        Start backup with restic
-        """
-        backup_command = [
-            "restic",
-            "-o",
-            "rclone.args=serve restic --stdio",
-            "-r",
-            f"rclone:backblaze:{self._repository_name}/sfbackup",
-            "--verbose",
-            "--json",
-            "backup",
-            "/var",
-        ]
-        with open("/var/backup.log", "w", encoding="utf-8") as log_file:
-            subprocess.Popen(
-                backup_command,
-                shell=False,
-                stdout=log_file,
-                stderr=subprocess.STDOUT,
-            )
-
-        self.state = ResticStates.BACKING_UP
-        self.progress = 0
-
-    def check_progress(self):
-        """
-        Check progress of ongoing backup operation
-        """
-        backup_status_check_command = ["tail", "-1", "/var/backup.log"]
-
-        if self.state in (ResticStates.NO_KEY, ResticStates.NOT_INITIALIZED):
-            return
-
-        # If the log file does not exists
-        if os.path.exists("/var/backup.log") is False:
-            self.state = ResticStates.INITIALIZED
-
-        with subprocess.Popen(
-            backup_status_check_command,
-            shell=False,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-        ) as backup_status_check_process_descriptor:
-            backup_process_status = (
-                backup_status_check_process_descriptor.communicate()[0].decode("utf-8")
-            )
-
-        try:
-            status = json.loads(backup_process_status)
-        except ValueError:
-            print(backup_process_status)
-            self.error_message = backup_process_status
-            return
-        if status["message_type"] == "status":
-            self.progress = status["percent_done"]
-            self.state = ResticStates.BACKING_UP
-        elif status["message_type"] == "summary":
-            self.state = ResticStates.INITIALIZED
-            self.progress = 0
-            self.snapshot_list.append(
-                {
-                    "short_id": status["snapshot_id"],
-                    # Current time in format 2021-12-02T00:02:51.086452543+03:00
-                    "time": datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f%z"),
-                }
-            )
-
-    def restore_from_backup(self, snapshot_id):
-        """
-        Restore from backup with restic
-        """
-        backup_restoration_command = [
-            "restic",
-            "-o",
-            "rclone.args=serve restic --stdio",
-            "-r",
-            f"rclone:backblaze:{self._repository_name}/sfbackup",
-            "restore",
-            snapshot_id,
-            "--target",
-            "/",
-        ]
-
-        self.state = ResticStates.RESTORING
-
-        subprocess.run(backup_restoration_command, shell=False)
-
-        self.state = ResticStates.INITIALIZED

selfprivacy_api/restic_controller/tasks.py

@@ -1,70 +0,0 @@
-"""Tasks for the restic controller."""
-from huey import crontab
-from selfprivacy_api.utils.huey import huey
-from . import ResticController, ResticStates
-
-
-@huey.task()
-def init_restic():
-    controller = ResticController()
-    if controller.state == ResticStates.NOT_INITIALIZED:
-        initialize_repository()
-
-
-@huey.task()
-def update_keys_from_userdata():
-    controller = ResticController()
-    controller.load_configuration()
-    controller.write_rclone_config()
-    initialize_repository()
-
-
-# Check every morning at 5:00 AM
-@huey.task(crontab(hour=5, minute=0))
-def cron_load_snapshots():
-    controller = ResticController()
-    controller.load_snapshots()
-
-
-# Check every morning at 5:00 AM
-@huey.task()
-def load_snapshots():
-    controller = ResticController()
-    controller.load_snapshots()
-    if controller.state == ResticStates.NOT_INITIALIZED:
-        load_snapshots.schedule(delay=120)
-
-
-@huey.task()
-def initialize_repository():
-    controller = ResticController()
-    if controller.state is not ResticStates.NO_KEY:
-        controller.initialize_repository()
-        load_snapshots()
-
-
-@huey.task()
-def fetch_backup_status():
-    controller = ResticController()
-    if controller.state is ResticStates.BACKING_UP:
-        controller.check_progress()
-        if controller.state is ResticStates.BACKING_UP:
-            fetch_backup_status.schedule(delay=2)
-        else:
-            load_snapshots.schedule(delay=240)
-
-
-@huey.task()
-def start_backup():
-    controller = ResticController()
-    if controller.state is ResticStates.NOT_INITIALIZED:
-        resp = initialize_repository()
-        resp.get()
-    controller.start_backup()
-    fetch_backup_status.schedule(delay=3)
-
-
-@huey.task()
-def restore_from_backup(snapshot):
-    controller = ResticController()
-    controller.restore_from_backup(snapshot)

selfprivacy_api/services/__init__.py

@@ -3,7 +3,7 @@
 import typing
 from selfprivacy_api.services.bitwarden import Bitwarden
 from selfprivacy_api.services.gitea import Gitea
-from selfprivacy_api.services.jitsi import Jitsi
+from selfprivacy_api.services.jitsimeet import JitsiMeet
 from selfprivacy_api.services.mailserver import MailServer
 from selfprivacy_api.services.nextcloud import Nextcloud
 from selfprivacy_api.services.pleroma import Pleroma
@@ -18,7 +18,7 @@ services: list[Service] = [
     Nextcloud(),
     Pleroma(),
     Ocserv(),
-    Jitsi(),
+    JitsiMeet(),
 ]
 
 
@@ -42,7 +42,7 @@ def get_disabled_services() -> list[Service]:
 
 
 def get_services_by_location(location: str) -> list[Service]:
-    return [service for service in services if service.get_location() == location]
+    return [service for service in services if service.get_drive() == location]
 
 
 def get_all_required_dns_records() -> list[ServiceDnsRecord]:
@@ -54,14 +54,20 @@ def get_all_required_dns_records() -> list[ServiceDnsRecord]:
             name="api",
             content=ip4,
             ttl=3600,
-        ),
-        ServiceDnsRecord(
-            type="AAAA",
-            name="api",
-            content=ip6,
-            ttl=3600,
+            display_name="SelfPrivacy API",
         ),
     ]
+
+    if ip6 is not None:
+        dns_records.append(
+            ServiceDnsRecord(
+                type="AAAA",
+                name="api",
+                content=ip6,
+                ttl=3600,
+                display_name="SelfPrivacy API (IPv6)",
+            )
+        )
     for service in get_enabled_services():
-        dns_records += service.get_dns_records()
+        dns_records += service.get_dns_records(ip4, ip6)
     return dns_records

selfprivacy_api/services/bitwarden/__init__.py

@@ -1,17 +1,12 @@
 """Class representing Bitwarden service"""
 import base64
 import subprocess
-import typing
+from typing import Optional, List
 
-from selfprivacy_api.jobs import Job, JobStatus, Jobs
-from selfprivacy_api.services.generic_service_mover import FolderMoveNames, move_service
-from selfprivacy_api.services.generic_size_counter import get_storage_usage
-from selfprivacy_api.services.generic_status_getter import get_service_status
-from selfprivacy_api.services.service import Service, ServiceDnsRecord, ServiceStatus
-from selfprivacy_api.utils import ReadUserData, WriteUserData, get_domain
-from selfprivacy_api.utils.block_devices import BlockDevice
-from selfprivacy_api.utils.huey import huey
-import selfprivacy_api.utils.network as network_utils
+from selfprivacy_api.utils import get_domain
+
+from selfprivacy_api.utils.systemd import get_service_status
+from selfprivacy_api.services.service import Service, ServiceStatus
 from selfprivacy_api.services.bitwarden.icon import BITWARDEN_ICON
 
 
@@ -39,11 +34,19 @@ class Bitwarden(Service):
         return base64.b64encode(BITWARDEN_ICON.encode("utf-8")).decode("utf-8")
 
     @staticmethod
-    def get_url() -> typing.Optional[str]:
+    def get_user() -> str:
+        return "vaultwarden"
+
+    @staticmethod
+    def get_url() -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://password.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "password"
+
     @staticmethod
     def is_movable() -> bool:
         return True
@@ -53,9 +56,8 @@ class Bitwarden(Service):
         return False
 
     @staticmethod
-    def is_enabled() -> bool:
-        with ReadUserData() as user_data:
-            return user_data.get("bitwarden", {}).get("enable", False)
+    def get_backup_description() -> str:
+        return "Password database, encryption certificate and attachments."
 
     @staticmethod
     def get_status() -> ServiceStatus:
@@ -70,22 +72,6 @@ class Bitwarden(Service):
         """
         return get_service_status("vaultwarden.service")
 
-    @staticmethod
-    def enable():
-        """Enable Bitwarden service."""
-        with WriteUserData() as user_data:
-            if "bitwarden" not in user_data:
-                user_data["bitwarden"] = {}
-            user_data["bitwarden"]["enable"] = True
-
-    @staticmethod
-    def disable():
-        """Disable Bitwarden service."""
-        with WriteUserData() as user_data:
-            if "bitwarden" not in user_data:
-                user_data["bitwarden"] = {}
-            user_data["bitwarden"]["enable"] = False
-
     @staticmethod
     def stop():
         subprocess.run(["systemctl", "stop", "vaultwarden.service"])
@@ -111,64 +97,5 @@ class Bitwarden(Service):
         return ""
 
     @staticmethod
-    def get_storage_usage() -> int:
-        storage_usage = 0
-        storage_usage += get_storage_usage("/var/lib/bitwarden")
-        storage_usage += get_storage_usage("/var/lib/bitwarden_rs")
-        return storage_usage
-
-    @staticmethod
-    def get_location() -> str:
-        with ReadUserData() as user_data:
-            if user_data.get("useBinds", False):
-                return user_data.get("bitwarden", {}).get("location", "sda1")
-            else:
-                return "sda1"
-
-    @staticmethod
-    def get_dns_records() -> typing.List[ServiceDnsRecord]:
-        """Return list of DNS records for Bitwarden service."""
-        return [
-            ServiceDnsRecord(
-                type="A",
-                name="password",
-                content=network_utils.get_ip4(),
-                ttl=3600,
-            ),
-            ServiceDnsRecord(
-                type="AAAA",
-                name="password",
-                content=network_utils.get_ip6(),
-                ttl=3600,
-            ),
-        ]
-
-    def move_to_volume(self, volume: BlockDevice) -> Job:
-        job = Jobs.get_instance().add(
-            type_id="services.bitwarden.move",
-            name="Move Bitwarden",
-            description=f"Moving Bitwarden data to {volume.name}",
-        )
-
-        move_service(
-            self,
-            volume,
-            job,
-            [
-                FolderMoveNames(
-                    name="bitwarden",
-                    bind_location="/var/lib/bitwarden",
-                    group="vaultwarden",
-                    owner="vaultwarden",
-                ),
-                FolderMoveNames(
-                    name="bitwarden_rs",
-                    bind_location="/var/lib/bitwarden_rs",
-                    group="vaultwarden",
-                    owner="vaultwarden",
-                ),
-            ],
-            "bitwarden",
-        )
-
-        return job
+    def get_folders() -> List[str]:
+        return ["/var/lib/bitwarden", "/var/lib/bitwarden_rs"]