Compare commits
No commits in common. "master" and "master" have entirely different histories.
|
|
@ -5,13 +5,13 @@ name: default
|
|||
steps:
|
||||
- name: Run Tests and Generate Coverage Report
|
||||
commands:
|
||||
- nix flake check -L
|
||||
- coverage run -m pytest -q
|
||||
- coverage xml
|
||||
- sonar-scanner -Dsonar.projectKey=SelfPrivacy-REST-API -Dsonar.sources=. -Dsonar.host.url=http://analyzer.lan:9000 -Dsonar.login="$SONARQUBE_TOKEN"
|
||||
environment:
|
||||
SONARQUBE_TOKEN:
|
||||
from_secret: SONARQUBE_TOKEN
|
||||
|
||||
|
||||
- name: Run Bandit Checks
|
||||
commands:
|
||||
- bandit -ll -r selfprivacy_api
|
||||
|
|
@ -22,7 +22,3 @@ steps:
|
|||
|
||||
node:
|
||||
server: builder
|
||||
|
||||
trigger:
|
||||
event:
|
||||
- push
|
||||
|
|
|
|||
4
.flake8
4
.flake8
|
|
@ -1,4 +0,0 @@
|
|||
[flake8]
|
||||
max-line-length = 80
|
||||
select = C,E,F,W,B,B950
|
||||
extend-ignore = E203, E501
|
||||
|
|
@ -147,7 +147,3 @@ cython_debug/
|
|||
# End of https://www.toptal.com/developers/gitignore/api/flask
|
||||
|
||||
*.db
|
||||
*.rdb
|
||||
|
||||
/result
|
||||
/.nixos-test-history
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
<component name="InspectionProjectProfileManager">
|
||||
<settings>
|
||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||
<version value="1.0" />
|
||||
</settings>
|
||||
</component>
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
|
||||
</project>
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/selfprivacy-rest-api.iml" filepath="$PROJECT_DIR$/.idea/selfprivacy-rest-api.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="jdk" jdkName="Python 3.9" jdkType="Python SDK" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
<component name="PyDocumentationSettings">
|
||||
<option name="format" value="PLAIN" />
|
||||
<option name="myDocStringFormat" value="Plain" />
|
||||
</component>
|
||||
<component name="TestRunnerService">
|
||||
<option name="PROJECT_TEST_RUNNER" value="py.test" />
|
||||
</component>
|
||||
</module>
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CommitMessageInspectionProfile">
|
||||
<profile version="1.0">
|
||||
<inspection_tool class="CommitFormat" enabled="true" level="WARNING" enabled_by_default="true" />
|
||||
<inspection_tool class="CommitNamingConvention" enabled="true" level="WARNING" enabled_by_default="true" />
|
||||
</profile>
|
||||
</component>
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,6 +1,3 @@
|
|||
[MASTER]
|
||||
init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))"
|
||||
extension-pkg-whitelist=pydantic
|
||||
|
||||
[FORMAT]
|
||||
max-line-length=88
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
# SelfPrivacy API contributors guide
|
||||
|
||||
Instructions for [VScode](https://code.visualstudio.com) or [VScodium](https://github.com/VSCodium/vscodium) under Unix-like platform.
|
||||
|
||||
1. **To get started, create an account for yourself on the** [**SelfPrivacy Gitea**](https://git.selfprivacy.org/user/sign_up). Proceed to fork
|
||||
the [repository](https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api), and clone it on your local computer:
|
||||
|
||||
```git clone https://git.selfprivacy.org/your_user_name/selfprivacy-rest-api```
|
||||
|
||||
2. **Install Nix**
|
||||
|
||||
```sh <(curl -L https://nixos.org/nix/install)```
|
||||
|
||||
For detailed installation information, please review and follow: [link](https://nixos.org/manual/nix/stable/installation/installing-binary.html#installing-a-binary-distribution).
|
||||
|
||||
3. **Change directory to the cloned repository and start a nix shell:**
|
||||
|
||||
```cd selfprivacy-rest-api && nix-shell```
|
||||
|
||||
Nix will install all of the necessary packages for development work, all further actions will take place only within nix-shell.
|
||||
|
||||
4. **Install these plugins for VScode/VScodium**
|
||||
|
||||
Required: ```ms-python.python```, ```ms-python.vscode-pylance```
|
||||
|
||||
Optional, but highly recommended: ```ms-python.black-formatter```, ```bbenoist.Nix```, ```ryanluker.vscode-coverage-gutters```
|
||||
|
||||
5. **Set the path to the python interpreter from the nix store.** To do this, execute the command:
|
||||
|
||||
```whereis python```
|
||||
|
||||
Copy the path that starts with ```/nix/store/``` and ends with ```env/bin/python```
|
||||
|
||||
```/nix/store/???-python3-3.9.??-env/bin/python```
|
||||
|
||||
Click on the python version selection in the lower right corner, and replace the path to the interpreter in the project with the one you copied from the terminal.
|
||||
|
||||
6. **Congratulations :) Now you can develop new changes and test the project locally in a Nix environment.**
|
||||
|
||||
## What do you need to know before starting development work?
|
||||
- RestAPI is no longer utilized, the project has moved to [GraphQL](https://graphql.org), however, the API functionality still works on Rest
|
||||
|
||||
|
||||
## What to do after making changes to the repository?
|
||||
|
||||
**Run unit tests** using ```pytest .```
|
||||
Make sure that all tests pass successfully and the API works correctly. For convenience, you can use the built-in VScode interface.
|
||||
|
||||
How to review the percentage of code coverage? Execute the command:
|
||||
|
||||
```coverage run -m pytest && coverage xml && coverage report```
|
||||
|
||||
Next, use the recommended extension ```ryanluker.vscode-coverage-gutters```, navigate to one of the test files, and click the "watch" button on the bottom panel of VScode.
|
||||
|
||||
**Format (linting) code**, we use [black](https://pypi.org/project/black/) formatting, enter
|
||||
```black .``` to automatically format files, or use the recommended extension.
|
||||
|
||||
**And please remember, we have adopted** [**commit naming convention**](https://www.conventionalcommits.org/en/v1.0.0/), follow the link for more information.
|
||||
|
||||
Please request a review from at least one of the other maintainers. If you are not sure who to request, request a review from SelfPrivacy/Devs team.
|
||||
|
||||
## Helpful links!
|
||||
|
||||
**SelfPrivacy Contributor chat :3**
|
||||
|
||||
- [**Telegram:** @selfprivacy_dev](https://t.me/selfprivacy_dev)
|
||||
- [**Matrix:** #dev:selfprivacy.org](https://matrix.to/#/#dev:selfprivacy.org)
|
||||
|
||||
**Helpful material to review:**
|
||||
|
||||
- [GraphQL Query Language Documentation](https://graphql.org/)
|
||||
- [Documentation Strawberry - python library for working with GraphQL](https://strawberry.rocks/docs/)
|
||||
- [Nix Documentation](https://nixos.org/guides/ad-hoc-developer-environments.html)
|
||||
|
||||
### Track your time
|
||||
|
||||
If you are working on a task, please track your time and add it to the commit message. For example:
|
||||
|
||||
```
|
||||
feat: add new feature
|
||||
|
||||
- did some work
|
||||
- did some more work
|
||||
|
||||
fixes #4, spent @1h30m
|
||||
```
|
||||
|
||||
[Timewarrior](https://timewarrior.net/) is a good tool for tracking time.
|
||||
92
README.md
92
README.md
|
|
@ -1,92 +0,0 @@
|
|||
# SelfPrivacy GraphQL API which allows app to control your server
|
||||
|
||||
|
||||
|
||||
## Build
|
||||
|
||||
```console
|
||||
$ nix build
|
||||
```
|
||||
|
||||
In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
|
||||
|
||||
## Develop
|
||||
|
||||
```console
|
||||
$ nix develop
|
||||
[SP devshell:/dir/selfprivacy-rest-api]$ python
|
||||
Python 3.10.13 (main, Aug 24 2023, 12:59:26) [GCC 12.3.0] on linux
|
||||
Type "help", "copyright", "credits" or "license" for more information.
|
||||
(ins)>>>
|
||||
```
|
||||
|
||||
If you don't have experimental flakes enabled, you can use the following command:
|
||||
|
||||
```console
|
||||
$ nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
|
||||
```console
|
||||
$ nix flake check -L
|
||||
```
|
||||
|
||||
Run the same test suite, but additionally create `./result/coverage.xml` in the current directory:
|
||||
```console
|
||||
$ nix build .#checks.x86_64-linux.default -L
|
||||
```
|
||||
|
||||
Alternatively, just print the path to `/nix/store/...coverage.xml` without creating any files in the current directory:
|
||||
```console
|
||||
$ nix build .#checks.x86_64-linux.default -L --print-out-paths --no-link
|
||||
```
|
||||
|
||||
Run the same test suite with arbitrary pytest options:
|
||||
```console
|
||||
$ pytest-vm.sh # specify pytest options here, e.g. `--last-failed`
|
||||
```
|
||||
When running using the script, pytest cache is preserved between runs in `.pytest_cache` folder.
|
||||
NixOS VM state temporary resides in `${TMPDIR:=/tmp}/nixos-vm-tmp-dir/vm-state-machine` during the test.
|
||||
Git workdir directory is shared read-write with VM via `.nixos-vm-tmp-dir/shared-xchg` symlink. VM accesses workdir contents via `/tmp/shared` mount point and `/root/source` symlink.
|
||||
|
||||
Launch VM and execute commands manually either in Linux console (user `root`) or using python NixOS tests driver API (refer to [NixOS documentation](https://nixos.org/manual/nixos/stable/#ssec-machine-objects)):
|
||||
```console
|
||||
$ nix run .#checks.x86_64-linux.default.driverInteractive
|
||||
```
|
||||
|
||||
You can add `--keep-vm-state` in order to keep VM state between runs:
|
||||
```console
|
||||
$ TMPDIR=".nixos-vm-tmp-dir" nix run .#checks.x86_64-linux.default.driverInteractive --keep-vm-state
|
||||
```
|
||||
|
||||
Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.
|
||||
|
||||
## Dependencies and Dependant Modules
|
||||
|
||||
This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.
|
||||
|
||||
In order to synchronize nixpkgs input with the same from selfprivacy-nixos-config repository, use this command:
|
||||
|
||||
```console
|
||||
$ nix flake lock --override-input nixpkgs nixpkgs --inputs-from git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
|
||||
```
|
||||
|
||||
Replace BRANCH with the branch name of selfprivacy-nixos-config repository you want to sync with. During development nixpkgs input update might be required in both selfprivacy-rest-api and selfprivacy-nixos-config repositories simultaneously. So, a new feature branch might be temporarily used until selfprivacy-nixos-config gets the feature branch merged.
|
||||
|
||||
Show current flake inputs (e.g. nixpkgs):
|
||||
```console
|
||||
$ nix flake metadata
|
||||
```
|
||||
|
||||
Show selfprivacy-nixos-config Nix flake inputs (including nixpkgs):
|
||||
```console
|
||||
$ nix flake metadata git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
|
||||
```
|
||||
|
||||
Nix code for NixOS service module for API is located in NixOS configuration repository.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
{ lib, python39Packages }:
|
||||
with python39Packages;
|
||||
buildPythonApplication {
|
||||
pname = "selfprivacy-api";
|
||||
version = "2.0.0";
|
||||
|
||||
propagatedBuildInputs = [
|
||||
setuptools
|
||||
portalocker
|
||||
pytz
|
||||
pytest
|
||||
pytest-mock
|
||||
pytest-datadir
|
||||
huey
|
||||
gevent
|
||||
mnemonic
|
||||
pydantic
|
||||
typing-extensions
|
||||
psutil
|
||||
fastapi
|
||||
uvicorn
|
||||
(buildPythonPackage rec {
|
||||
pname = "strawberry-graphql";
|
||||
version = "0.123.0";
|
||||
format = "pyproject";
|
||||
patches = [
|
||||
./strawberry-graphql.patch
|
||||
];
|
||||
propagatedBuildInputs = [
|
||||
typing-extensions
|
||||
python-multipart
|
||||
python-dateutil
|
||||
# flask
|
||||
pydantic
|
||||
pygments
|
||||
poetry
|
||||
# flask-cors
|
||||
(buildPythonPackage rec {
|
||||
pname = "graphql-core";
|
||||
version = "3.2.0";
|
||||
format = "setuptools";
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84=";
|
||||
};
|
||||
checkInputs = [
|
||||
pytest-asyncio
|
||||
pytest-benchmark
|
||||
pytestCheckHook
|
||||
];
|
||||
pythonImportsCheck = [
|
||||
"graphql"
|
||||
];
|
||||
})
|
||||
];
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo=";
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
src = ./.;
|
||||
}
|
||||
31
default.nix
31
default.nix
|
|
@ -1,29 +1,2 @@
|
|||
{ pythonPackages, rev ? "local" }:
|
||||
|
||||
pythonPackages.buildPythonPackage rec {
|
||||
pname = "selfprivacy-graphql-api";
|
||||
version = rev;
|
||||
src = builtins.filterSource (p: t: p != ".git" && t != "symlink") ./.;
|
||||
propagatedBuildInputs = with pythonPackages; [
|
||||
fastapi
|
||||
gevent
|
||||
huey
|
||||
mnemonic
|
||||
portalocker
|
||||
psutil
|
||||
pydantic
|
||||
pytz
|
||||
redis
|
||||
setuptools
|
||||
strawberry-graphql
|
||||
typing-extensions
|
||||
uvicorn
|
||||
];
|
||||
pythonImportsCheck = [ "selfprivacy_api" ];
|
||||
doCheck = false;
|
||||
meta = {
|
||||
description = ''
|
||||
SelfPrivacy Server Management API
|
||||
'';
|
||||
};
|
||||
}
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
pkgs.callPackage ./api.nix {}
|
||||
|
|
|
|||
26
flake.lock
26
flake.lock
|
|
@ -1,26 +0,0 @@
|
|||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1709677081,
|
||||
"narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
162
flake.nix
162
flake.nix
|
|
@ -1,162 +0,0 @@
|
|||
{
|
||||
description = "SelfPrivacy API flake";
|
||||
|
||||
inputs.nixpkgs.url = "github:nixos/nixpkgs";
|
||||
|
||||
outputs = { self, nixpkgs, ... }:
|
||||
let
|
||||
system = "x86_64-linux";
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
selfprivacy-graphql-api = pkgs.callPackage ./default.nix {
|
||||
pythonPackages = pkgs.python310Packages;
|
||||
rev = self.shortRev or self.dirtyShortRev or "dirty";
|
||||
};
|
||||
python = self.packages.${system}.default.pythonModule;
|
||||
python-env =
|
||||
python.withPackages (ps:
|
||||
self.packages.${system}.default.propagatedBuildInputs ++ (with ps; [
|
||||
coverage
|
||||
pytest
|
||||
pytest-datadir
|
||||
pytest-mock
|
||||
pytest-subprocess
|
||||
black
|
||||
mypy
|
||||
pylsp-mypy
|
||||
python-lsp-black
|
||||
python-lsp-server
|
||||
pyflakes
|
||||
typer # for strawberry
|
||||
types-redis # for mypy
|
||||
] ++ strawberry-graphql.optional-dependencies.cli));
|
||||
|
||||
vmtest-src-dir = "/root/source";
|
||||
shellMOTD = ''
|
||||
Welcome to SP API development shell!
|
||||
|
||||
[formatters]
|
||||
|
||||
black
|
||||
nixpkgs-fmt
|
||||
|
||||
[testing in NixOS VM]
|
||||
|
||||
nixos-test-driver - run an interactive NixOS VM with all dependencies included and 2 disk volumes
|
||||
pytest-vm - run pytest in an ephemeral NixOS VM with Redis, accepting pytest arguments
|
||||
'';
|
||||
in
|
||||
{
|
||||
# see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
|
||||
packages.${system} = {
|
||||
default = selfprivacy-graphql-api;
|
||||
pytest-vm = pkgs.writeShellScriptBin "pytest-vm" ''
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o xtrace
|
||||
|
||||
# see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
|
||||
export TMPDIR=''${TMPDIR:=/tmp}/nixos-vm-tmp-dir
|
||||
readonly NIXOS_VM_SHARED_DIR_HOST="$TMPDIR/shared-xchg"
|
||||
readonly NIXOS_VM_SHARED_DIR_GUEST="/tmp/shared"
|
||||
|
||||
mkdir -p "$TMPDIR"
|
||||
ln -sfv "$PWD" -T "$NIXOS_VM_SHARED_DIR_HOST"
|
||||
|
||||
SCRIPT=$(cat <<EOF
|
||||
start_all()
|
||||
machine.succeed("ln -sf $NIXOS_VM_SHARED_DIR_GUEST -T ${vmtest-src-dir} >&2")
|
||||
machine.succeed("cd ${vmtest-src-dir} && coverage run -m pytest -v $@ >&2")
|
||||
machine.succeed("cd ${vmtest-src-dir} && coverage report >&2")
|
||||
EOF
|
||||
)
|
||||
|
||||
if [ -f "/etc/arch-release" ]; then
|
||||
${self.checks.${system}.default.driverInteractive}/bin/nixos-test-driver --no-interactive <(printf "%s" "$SCRIPT")
|
||||
else
|
||||
${self.checks.${system}.default.driver}/bin/nixos-test-driver -- <(printf "%s" "$SCRIPT")
|
||||
fi
|
||||
'';
|
||||
};
|
||||
nixosModules.default =
|
||||
import ./nixos/module.nix self.packages.${system}.default;
|
||||
devShells.${system}.default = pkgs.mkShellNoCC {
|
||||
name = "SP API dev shell";
|
||||
packages = with pkgs; [
|
||||
nixpkgs-fmt
|
||||
rclone
|
||||
redis
|
||||
restic
|
||||
self.packages.${system}.pytest-vm
|
||||
# FIXME consider loading this explicitly only after ArchLinux issue is solved
|
||||
self.checks.x86_64-linux.default.driverInteractive
|
||||
# the target API application python environment
|
||||
python-env
|
||||
];
|
||||
shellHook = ''
|
||||
# envs set with export and as attributes are treated differently.
|
||||
# for example. printenv <Name> will not fetch the value of an attribute.
|
||||
export TEST_MODE="true"
|
||||
|
||||
# more tips for bash-completion to work on non-NixOS:
|
||||
# https://discourse.nixos.org/t/whats-the-nix-way-of-bash-completion-for-packages/20209/16?u=alexoundos
|
||||
# Load installed profiles
|
||||
for file in "/etc/profile.d/"*.sh; do
|
||||
# If that folder doesn't exist, bash loves to return the whole glob
|
||||
[[ -f "$file" ]] && source "$file"
|
||||
done
|
||||
|
||||
printf "%s" "${shellMOTD}"
|
||||
'';
|
||||
};
|
||||
checks.${system} = {
|
||||
fmt-check = pkgs.runCommandLocal "sp-api-fmt-check"
|
||||
{ nativeBuildInputs = [ pkgs.black ]; }
|
||||
"black --check ${self.outPath} > $out";
|
||||
default =
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "default";
|
||||
nodes.machine = { lib, pkgs, ... }: {
|
||||
# 2 additional disks (1024 MiB and 200 MiB) with empty ext4 FS
|
||||
virtualisation.emptyDiskImages = [ 1024 200 ];
|
||||
virtualisation.fileSystems."/volumes/vdb" = {
|
||||
autoFormat = true;
|
||||
device = "/dev/vdb"; # this name is chosen by QEMU, not here
|
||||
fsType = "ext4";
|
||||
noCheck = true;
|
||||
};
|
||||
virtualisation.fileSystems."/volumes/vdc" = {
|
||||
autoFormat = true;
|
||||
device = "/dev/vdc"; # this name is chosen by QEMU, not here
|
||||
fsType = "ext4";
|
||||
noCheck = true;
|
||||
};
|
||||
boot.consoleLogLevel = lib.mkForce 3;
|
||||
documentation.enable = false;
|
||||
services.journald.extraConfig = lib.mkForce "";
|
||||
services.redis.servers.sp-api = {
|
||||
enable = true;
|
||||
save = [ ];
|
||||
settings.notify-keyspace-events = "KEA";
|
||||
};
|
||||
environment.systemPackages = with pkgs; [
|
||||
python-env
|
||||
# TODO: these can be passed via wrapper script around app
|
||||
rclone
|
||||
restic
|
||||
];
|
||||
environment.variables.TEST_MODE = "true";
|
||||
systemd.tmpfiles.settings.src.${vmtest-src-dir}.L.argument =
|
||||
self.outPath;
|
||||
};
|
||||
testScript = ''
|
||||
start_all()
|
||||
machine.succeed("cd ${vmtest-src-dir} && coverage run --data-file=/tmp/.coverage -m pytest -p no:cacheprovider -v >&2")
|
||||
machine.succeed("coverage xml --rcfile=${vmtest-src-dir}/.coveragerc --data-file=/tmp/.coverage >&2")
|
||||
machine.copy_from_vm("coverage.xml", ".")
|
||||
machine.succeed("coverage report >&2")
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
nixConfig.bash-prompt = ''\n\[\e[1;32m\][\[\e[0m\]\[\e[1;34m\]SP devshell\[\e[0m\]\[\e[1;32m\]:\w]\$\[\[\e[0m\] '';
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
@startuml
|
||||
|
||||
left to right direction
|
||||
|
||||
title repositories and flake inputs relations diagram
|
||||
|
||||
cloud nixpkgs as nixpkgs_transit
|
||||
control "<font:monospaced><size:15>nixos-rebuild" as nixos_rebuild
|
||||
component "SelfPrivacy\nAPI app" as selfprivacy_app
|
||||
component "SelfPrivacy\nNixOS configuration" as nixos_configuration
|
||||
|
||||
note top of nixos_configuration : SelfPrivacy\nAPI service module
|
||||
|
||||
nixos_configuration ).. nixpkgs_transit
|
||||
nixpkgs_transit ..> selfprivacy_app
|
||||
selfprivacy_app --> nixos_configuration
|
||||
[nixpkgs] --> nixos_configuration
|
||||
nixos_configuration -> nixos_rebuild
|
||||
|
||||
footer %date("yyyy-MM-dd'T'HH:mmZ")
|
||||
|
||||
@enduml
|
||||
166
nixos/module.nix
166
nixos/module.nix
|
|
@ -1,166 +0,0 @@
|
|||
selfprivacy-graphql-api: { config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.selfprivacy-api;
|
||||
config-id = "default";
|
||||
nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
|
||||
nix = "${config.nix.package.out}/bin/nix";
|
||||
in
|
||||
{
|
||||
options.services.selfprivacy-api = {
|
||||
enable = lib.mkOption {
|
||||
default = true;
|
||||
type = lib.types.bool;
|
||||
description = ''
|
||||
Enable SelfPrivacy API service
|
||||
'';
|
||||
};
|
||||
};
|
||||
config = lib.mkIf cfg.enable {
|
||||
users.users."selfprivacy-api" = {
|
||||
isNormalUser = false;
|
||||
isSystemUser = true;
|
||||
extraGroups = [ "opendkim" ];
|
||||
group = "selfprivacy-api";
|
||||
};
|
||||
users.groups."selfprivacy-api".members = [ "selfprivacy-api" ];
|
||||
|
||||
systemd.services.selfprivacy-api = {
|
||||
description = "API Server used to control system from the mobile application";
|
||||
environment = config.nix.envVars // {
|
||||
HOME = "/root";
|
||||
PYTHONUNBUFFERED = "1";
|
||||
} // config.networking.proxy.envVars;
|
||||
path = [
|
||||
"/var/"
|
||||
"/var/dkim/"
|
||||
pkgs.coreutils
|
||||
pkgs.gnutar
|
||||
pkgs.xz.bin
|
||||
pkgs.gzip
|
||||
pkgs.gitMinimal
|
||||
config.nix.package.out
|
||||
pkgs.restic
|
||||
pkgs.mkpasswd
|
||||
pkgs.util-linux
|
||||
pkgs.e2fsprogs
|
||||
pkgs.iproute2
|
||||
];
|
||||
after = [ "network-online.target" ];
|
||||
wantedBy = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
User = "root";
|
||||
ExecStart = "${selfprivacy-graphql-api}/bin/app.py";
|
||||
Restart = "always";
|
||||
RestartSec = "5";
|
||||
};
|
||||
};
|
||||
systemd.services.selfprivacy-api-worker = {
|
||||
description = "Task worker for SelfPrivacy API";
|
||||
environment = config.nix.envVars // {
|
||||
HOME = "/root";
|
||||
PYTHONUNBUFFERED = "1";
|
||||
PYTHONPATH =
|
||||
pkgs.python310Packages.makePythonPath [ selfprivacy-graphql-api ];
|
||||
} // config.networking.proxy.envVars;
|
||||
path = [
|
||||
"/var/"
|
||||
"/var/dkim/"
|
||||
pkgs.coreutils
|
||||
pkgs.gnutar
|
||||
pkgs.xz.bin
|
||||
pkgs.gzip
|
||||
pkgs.gitMinimal
|
||||
config.nix.package.out
|
||||
pkgs.restic
|
||||
pkgs.mkpasswd
|
||||
pkgs.util-linux
|
||||
pkgs.e2fsprogs
|
||||
pkgs.iproute2
|
||||
];
|
||||
after = [ "network-online.target" ];
|
||||
wantedBy = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
User = "root";
|
||||
ExecStart = "${pkgs.python310Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey";
|
||||
Restart = "always";
|
||||
RestartSec = "5";
|
||||
};
|
||||
};
|
||||
# One shot systemd service to rebuild NixOS using nixos-rebuild
|
||||
systemd.services.sp-nixos-rebuild = {
|
||||
description = "nixos-rebuild switch";
|
||||
environment = config.nix.envVars // {
|
||||
HOME = "/root";
|
||||
} // config.networking.proxy.envVars;
|
||||
# TODO figure out how to get dependencies list reliably
|
||||
path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
|
||||
# TODO set proper timeout for reboot instead of service restart
|
||||
serviceConfig = {
|
||||
User = "root";
|
||||
WorkingDirectory = "/etc/nixos";
|
||||
# sync top-level flake with sp-modules sub-flake
|
||||
# (https://github.com/NixOS/nix/issues/9339)
|
||||
ExecStartPre = ''
|
||||
${nix} flake lock --override-input sp-modules path:./sp-modules
|
||||
'';
|
||||
ExecStart = ''
|
||||
${nixos-rebuild} switch --flake .#${config-id}
|
||||
'';
|
||||
KillMode = "none";
|
||||
SendSIGKILL = "no";
|
||||
};
|
||||
restartIfChanged = false;
|
||||
unitConfig.X-StopOnRemoval = false;
|
||||
};
|
||||
# One shot systemd service to upgrade NixOS using nixos-rebuild
|
||||
systemd.services.sp-nixos-upgrade = {
|
||||
# protection against simultaneous runs
|
||||
after = [ "sp-nixos-rebuild.service" ];
|
||||
description = "Upgrade NixOS and SP modules to latest versions";
|
||||
environment = config.nix.envVars // {
|
||||
HOME = "/root";
|
||||
} // config.networking.proxy.envVars;
|
||||
# TODO figure out how to get dependencies list reliably
|
||||
path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
|
||||
serviceConfig = {
|
||||
User = "root";
|
||||
WorkingDirectory = "/etc/nixos";
|
||||
# TODO get URL from systemd template parameter?
|
||||
ExecStartPre = ''
|
||||
${nix} flake update \
|
||||
--override-input selfprivacy-nixos-config git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes
|
||||
'';
|
||||
ExecStart = ''
|
||||
${nixos-rebuild} switch --flake .#${config-id}
|
||||
'';
|
||||
KillMode = "none";
|
||||
SendSIGKILL = "no";
|
||||
};
|
||||
restartIfChanged = false;
|
||||
unitConfig.X-StopOnRemoval = false;
|
||||
};
|
||||
# One shot systemd service to rollback NixOS using nixos-rebuild
|
||||
systemd.services.sp-nixos-rollback = {
|
||||
# protection against simultaneous runs
|
||||
after = [ "sp-nixos-rebuild.service" "sp-nixos-upgrade.service" ];
|
||||
description = "Rollback NixOS using nixos-rebuild";
|
||||
environment = config.nix.envVars // {
|
||||
HOME = "/root";
|
||||
} // config.networking.proxy.envVars;
|
||||
# TODO figure out how to get dependencies list reliably
|
||||
path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
|
||||
serviceConfig = {
|
||||
User = "root";
|
||||
WorkingDirectory = "/etc/nixos";
|
||||
ExecStart = ''
|
||||
${nixos-rebuild} switch --rollback --flake .#${config-id}
|
||||
'';
|
||||
KillMode = "none";
|
||||
SendSIGKILL = "no";
|
||||
};
|
||||
restartIfChanged = false;
|
||||
unitConfig.X-StopOnRemoval = false;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,24 +1,21 @@
|
|||
"""
|
||||
App tokens actions.
|
||||
The only actions on tokens that are accessible from APIs
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
"""App tokens actions"""
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
from mnemonic import Mnemonic
|
||||
|
||||
from selfprivacy_api.utils.timeutils import ensure_tz_aware, ensure_tz_aware_strict
|
||||
from selfprivacy_api.repositories.tokens.redis_tokens_repository import (
|
||||
RedisTokensRepository,
|
||||
)
|
||||
from selfprivacy_api.repositories.tokens.exceptions import (
|
||||
TokenNotFound,
|
||||
RecoveryKeyNotFound,
|
||||
InvalidMnemonic,
|
||||
NewDeviceKeyNotFound,
|
||||
)
|
||||
|
||||
TOKEN_REPO = RedisTokensRepository()
|
||||
from selfprivacy_api.utils.auth import (
|
||||
delete_token,
|
||||
generate_recovery_token,
|
||||
get_recovery_token_status,
|
||||
get_tokens_info,
|
||||
is_recovery_token_exists,
|
||||
is_recovery_token_valid,
|
||||
is_token_name_exists,
|
||||
is_token_name_pair_valid,
|
||||
refresh_token,
|
||||
get_token_name,
|
||||
)
|
||||
|
||||
|
||||
class TokenInfoWithIsCaller(BaseModel):
|
||||
|
|
@ -29,33 +26,20 @@ class TokenInfoWithIsCaller(BaseModel):
|
|||
is_caller: bool
|
||||
|
||||
|
||||
def _naive(date_time: datetime) -> datetime:
|
||||
if date_time is None:
|
||||
return None
|
||||
if date_time.tzinfo is not None:
|
||||
date_time.astimezone(timezone.utc)
|
||||
return date_time.replace(tzinfo=None)
|
||||
|
||||
|
||||
def get_api_tokens_with_caller_flag(caller_token: str) -> list[TokenInfoWithIsCaller]:
|
||||
"""Get the tokens info"""
|
||||
caller_name = TOKEN_REPO.get_token_by_token_string(caller_token).device_name
|
||||
tokens = TOKEN_REPO.get_tokens()
|
||||
caller_name = get_token_name(caller_token)
|
||||
tokens = get_tokens_info()
|
||||
return [
|
||||
TokenInfoWithIsCaller(
|
||||
name=token.device_name,
|
||||
date=token.created_at,
|
||||
is_caller=token.device_name == caller_name,
|
||||
name=token.name,
|
||||
date=token.date,
|
||||
is_caller=token.name == caller_name,
|
||||
)
|
||||
for token in tokens
|
||||
]
|
||||
|
||||
|
||||
def is_token_valid(token) -> bool:
|
||||
"""Check if token is valid"""
|
||||
return TOKEN_REPO.is_token_valid(token)
|
||||
|
||||
|
||||
class NotFoundException(Exception):
|
||||
"""Not found exception"""
|
||||
|
||||
|
|
@ -66,22 +50,19 @@ class CannotDeleteCallerException(Exception):
|
|||
|
||||
def delete_api_token(caller_token: str, token_name: str) -> None:
|
||||
"""Delete the token"""
|
||||
if TOKEN_REPO.is_token_name_pair_valid(token_name, caller_token):
|
||||
if is_token_name_pair_valid(token_name, caller_token):
|
||||
raise CannotDeleteCallerException("Cannot delete caller's token")
|
||||
if not TOKEN_REPO.is_token_name_exists(token_name):
|
||||
if not is_token_name_exists(token_name):
|
||||
raise NotFoundException("Token not found")
|
||||
token = TOKEN_REPO.get_token_by_name(token_name)
|
||||
TOKEN_REPO.delete_token(token)
|
||||
delete_token(token_name)
|
||||
|
||||
|
||||
def refresh_api_token(caller_token: str) -> str:
|
||||
"""Refresh the token"""
|
||||
try:
|
||||
old_token = TOKEN_REPO.get_token_by_token_string(caller_token)
|
||||
new_token = TOKEN_REPO.refresh_token(old_token)
|
||||
except TokenNotFound:
|
||||
new_token = refresh_token(caller_token)
|
||||
if new_token is None:
|
||||
raise NotFoundException("Token not found")
|
||||
return new_token.token
|
||||
return new_token
|
||||
|
||||
|
||||
class RecoveryTokenStatus(BaseModel):
|
||||
|
|
@ -95,23 +76,19 @@ class RecoveryTokenStatus(BaseModel):
|
|||
|
||||
|
||||
def get_api_recovery_token_status() -> RecoveryTokenStatus:
|
||||
"""Get the recovery token status, timezone-aware"""
|
||||
token = TOKEN_REPO.get_recovery_key()
|
||||
if token is None:
|
||||
"""Get the recovery token status"""
|
||||
if not is_recovery_token_exists():
|
||||
return RecoveryTokenStatus(exists=False, valid=False)
|
||||
is_valid = TOKEN_REPO.is_recovery_key_valid()
|
||||
|
||||
# New tokens are tz-aware, but older ones might not be
|
||||
expiry_date = token.expires_at
|
||||
if expiry_date is not None:
|
||||
expiry_date = ensure_tz_aware_strict(expiry_date)
|
||||
|
||||
status = get_recovery_token_status()
|
||||
if status is None:
|
||||
return RecoveryTokenStatus(exists=False, valid=False)
|
||||
is_valid = is_recovery_token_valid()
|
||||
return RecoveryTokenStatus(
|
||||
exists=True,
|
||||
valid=is_valid,
|
||||
date=ensure_tz_aware_strict(token.created_at),
|
||||
expiration=expiry_date,
|
||||
uses_left=token.uses_left,
|
||||
date=status["date"],
|
||||
expiration=status["expiration"],
|
||||
uses_left=status["uses_left"],
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -128,54 +105,12 @@ def get_new_api_recovery_key(
|
|||
) -> str:
|
||||
"""Get new recovery key"""
|
||||
if expiration_date is not None:
|
||||
expiration_date = ensure_tz_aware(expiration_date)
|
||||
current_time = datetime.now(timezone.utc)
|
||||
if expiration_date < current_time:
|
||||
current_time = datetime.now().timestamp()
|
||||
if expiration_date.timestamp() < current_time:
|
||||
raise InvalidExpirationDate("Expiration date is in the past")
|
||||
if uses_left is not None:
|
||||
if uses_left <= 0:
|
||||
raise InvalidUsesLeft("Uses must be greater than 0")
|
||||
|
||||
key = TOKEN_REPO.create_recovery_key(expiration_date, uses_left)
|
||||
mnemonic_phrase = Mnemonic(language="english").to_mnemonic(bytes.fromhex(key.key))
|
||||
return mnemonic_phrase
|
||||
|
||||
|
||||
def use_mnemonic_recovery_token(mnemonic_phrase, name):
|
||||
"""Use the recovery token by converting the mnemonic word list to a byte array.
|
||||
If the recovery token if invalid itself, return None
|
||||
If the binary representation of phrase not matches
|
||||
the byte array of the recovery token, return None.
|
||||
If the mnemonic phrase is valid then generate a device token and return it.
|
||||
Substract 1 from uses_left if it exists.
|
||||
mnemonic_phrase is a string representation of the mnemonic word list.
|
||||
"""
|
||||
try:
|
||||
token = TOKEN_REPO.use_mnemonic_recovery_key(mnemonic_phrase, name)
|
||||
return token.token
|
||||
except (RecoveryKeyNotFound, InvalidMnemonic):
|
||||
return None
|
||||
|
||||
|
||||
def delete_new_device_auth_token() -> None:
|
||||
TOKEN_REPO.delete_new_device_key()
|
||||
|
||||
|
||||
def get_new_device_auth_token() -> str:
|
||||
"""Generate and store a new device auth token which is valid for 10 minutes
|
||||
and return a mnemonic phrase representation
|
||||
"""
|
||||
key = TOKEN_REPO.get_new_device_key()
|
||||
return Mnemonic(language="english").to_mnemonic(bytes.fromhex(key.key))
|
||||
|
||||
|
||||
def use_new_device_auth_token(mnemonic_phrase, name) -> Optional[str]:
|
||||
"""Use the new device auth token by converting the mnemonic string to a byte array.
|
||||
If the mnemonic phrase is valid then generate a device token and return it.
|
||||
New device auth token must be deleted.
|
||||
"""
|
||||
try:
|
||||
token = TOKEN_REPO.use_mnemonic_new_device_key(mnemonic_phrase, name)
|
||||
return token.token
|
||||
except (NewDeviceKeyNotFound, InvalidMnemonic):
|
||||
return None
|
||||
key = generate_recovery_token(expiration_date, uses_left)
|
||||
return key
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
from selfprivacy_api.utils.block_devices import BlockDevices
|
||||
from selfprivacy_api.jobs import Jobs, Job
|
||||
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
from selfprivacy_api.services.tasks import move_service as move_service_task
|
||||
|
||||
|
||||
class ServiceNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class VolumeNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def move_service(service_id: str, volume_name: str) -> Job:
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
raise ServiceNotFoundError(f"No such service:{service_id}")
|
||||
|
||||
volume = BlockDevices().get_block_device(volume_name)
|
||||
if volume is None:
|
||||
raise VolumeNotFoundError(f"No such volume:{volume_name}")
|
||||
|
||||
service.assert_can_move(volume)
|
||||
|
||||
job = Jobs.add(
|
||||
type_id=f"services.{service.get_id()}.move",
|
||||
name=f"Move {service.get_display_name()}",
|
||||
description=f"Moving {service.get_display_name()} data to {volume.name}",
|
||||
)
|
||||
|
||||
move_service_task(service, volume, job)
|
||||
return job
|
||||
|
|
@ -31,7 +31,7 @@ def get_ssh_settings() -> UserdataSshSettings:
|
|||
if "enable" not in data["ssh"]:
|
||||
data["ssh"]["enable"] = True
|
||||
if "passwordAuthentication" not in data["ssh"]:
|
||||
data["ssh"]["passwordAuthentication"] = False
|
||||
data["ssh"]["passwordAuthentication"] = True
|
||||
if "rootKeys" not in data["ssh"]:
|
||||
data["ssh"]["rootKeys"] = []
|
||||
return UserdataSshSettings(**data["ssh"])
|
||||
|
|
@ -49,6 +49,19 @@ def set_ssh_settings(
|
|||
data["ssh"]["passwordAuthentication"] = password_authentication
|
||||
|
||||
|
||||
def add_root_ssh_key(public_key: str):
|
||||
with WriteUserData() as data:
|
||||
if "ssh" not in data:
|
||||
data["ssh"] = {}
|
||||
if "rootKeys" not in data["ssh"]:
|
||||
data["ssh"]["rootKeys"] = []
|
||||
# Return 409 if key already in array
|
||||
for key in data["ssh"]["rootKeys"]:
|
||||
if key == public_key:
|
||||
raise KeyAlreadyExists()
|
||||
data["ssh"]["rootKeys"].append(public_key)
|
||||
|
||||
|
||||
class KeyAlreadyExists(Exception):
|
||||
"""Key already exists"""
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,8 @@
|
|||
import os
|
||||
import subprocess
|
||||
import pytz
|
||||
from typing import Optional, List
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
from selfprivacy_api.jobs import Job, JobStatus, Jobs
|
||||
from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
|
||||
|
||||
from selfprivacy_api.utils import WriteUserData, ReadUserData
|
||||
|
||||
|
|
@ -15,7 +13,7 @@ def get_timezone() -> str:
|
|||
with ReadUserData() as user_data:
|
||||
if "timezone" in user_data:
|
||||
return user_data["timezone"]
|
||||
return "Etc/UTC"
|
||||
return "Europe/Uzhgorod"
|
||||
|
||||
|
||||
class InvalidTimezone(Exception):
|
||||
|
|
@ -60,68 +58,36 @@ def set_auto_upgrade_settings(
|
|||
user_data["autoUpgrade"]["allowReboot"] = allowReboot
|
||||
|
||||
|
||||
class ShellException(Exception):
|
||||
"""Something went wrong when calling another process"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def run_blocking(cmd: List[str], new_session: bool = False) -> str:
|
||||
"""Run a process, block until done, return output, complain if failed"""
|
||||
process_handle = subprocess.Popen(
|
||||
cmd,
|
||||
shell=False,
|
||||
start_new_session=new_session,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout_raw, stderr_raw = process_handle.communicate()
|
||||
stdout = stdout_raw.decode("utf-8")
|
||||
if stderr_raw is not None:
|
||||
stderr = stderr_raw.decode("utf-8")
|
||||
else:
|
||||
stderr = ""
|
||||
output = stdout + "\n" + stderr
|
||||
if process_handle.returncode != 0:
|
||||
raise ShellException(
|
||||
f"Shell command failed, command array: {cmd}, output: {output}"
|
||||
)
|
||||
return stdout
|
||||
|
||||
|
||||
def rebuild_system() -> Job:
|
||||
def rebuild_system() -> int:
|
||||
"""Rebuild the system"""
|
||||
job = Jobs.add(
|
||||
type_id="system.nixos.rebuild",
|
||||
name="Rebuild system",
|
||||
description="Applying the new system configuration by building the new NixOS generation.",
|
||||
status=JobStatus.CREATED,
|
||||
rebuild_result = subprocess.Popen(
|
||||
["systemctl", "start", "sp-nixos-rebuild.service"], start_new_session=True
|
||||
)
|
||||
rebuild_system_task(job)
|
||||
return job
|
||||
rebuild_result.communicate()[0]
|
||||
return rebuild_result.returncode
|
||||
|
||||
|
||||
def rollback_system() -> int:
|
||||
"""Rollback the system"""
|
||||
run_blocking(["systemctl", "start", "sp-nixos-rollback.service"], new_session=True)
|
||||
return 0
|
||||
|
||||
|
||||
def upgrade_system() -> Job:
|
||||
"""Upgrade the system"""
|
||||
job = Jobs.add(
|
||||
type_id="system.nixos.upgrade",
|
||||
name="Upgrade system",
|
||||
description="Upgrading the system to the latest version.",
|
||||
status=JobStatus.CREATED,
|
||||
rollback_result = subprocess.Popen(
|
||||
["systemctl", "start", "sp-nixos-rollback.service"], start_new_session=True
|
||||
)
|
||||
rebuild_system_task(job, upgrade=True)
|
||||
return job
|
||||
rollback_result.communicate()[0]
|
||||
return rollback_result.returncode
|
||||
|
||||
|
||||
def upgrade_system() -> int:
|
||||
"""Upgrade the system"""
|
||||
upgrade_result = subprocess.Popen(
|
||||
["systemctl", "start", "sp-nixos-upgrade.service"], start_new_session=True
|
||||
)
|
||||
upgrade_result.communicate()[0]
|
||||
return upgrade_result.returncode
|
||||
|
||||
|
||||
def reboot_system() -> None:
|
||||
"""Reboot the system"""
|
||||
run_blocking(["reboot"], new_session=True)
|
||||
subprocess.Popen(["reboot"], start_new_session=True)
|
||||
|
||||
|
||||
def get_system_version() -> str:
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ def get_users(
|
|||
)
|
||||
for user in user_data["users"]
|
||||
]
|
||||
if not exclude_primary and "username" in user_data.keys():
|
||||
if not exclude_primary:
|
||||
users.append(
|
||||
UserDataUser(
|
||||
username=user_data["username"],
|
||||
|
|
@ -107,12 +107,6 @@ class PasswordIsEmpty(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class InvalidConfiguration(Exception):
|
||||
"""The userdata is broken"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def create_user(username: str, password: str):
|
||||
if password == "":
|
||||
raise PasswordIsEmpty("Password is empty")
|
||||
|
|
@ -130,10 +124,6 @@ def create_user(username: str, password: str):
|
|||
|
||||
with ReadUserData() as user_data:
|
||||
ensure_ssh_and_users_fields_exist(user_data)
|
||||
if "username" not in user_data.keys():
|
||||
raise InvalidConfiguration(
|
||||
"Broken config: Admin name is not defined. Consider recovery or add it manually"
|
||||
)
|
||||
if username == user_data["username"]:
|
||||
raise UserAlreadyExists("User already exists")
|
||||
if username in [user["username"] for user in user_data["users"]]:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,14 @@ import uvicorn
|
|||
from selfprivacy_api.dependencies import get_api_version
|
||||
from selfprivacy_api.graphql.schema import schema
|
||||
from selfprivacy_api.migrations import run_migrations
|
||||
from selfprivacy_api.restic_controller.tasks import init_restic
|
||||
|
||||
from selfprivacy_api.rest import (
|
||||
system,
|
||||
users,
|
||||
api_auth,
|
||||
services,
|
||||
)
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
|
@ -26,6 +33,10 @@ app.add_middleware(
|
|||
)
|
||||
|
||||
|
||||
app.include_router(system.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(api_auth.router)
|
||||
app.include_router(services.router)
|
||||
app.include_router(graphql_app, prefix="/graphql")
|
||||
|
||||
|
||||
|
|
@ -38,6 +49,7 @@ async def get_version():
|
|||
@app.on_event("startup")
|
||||
async def startup():
|
||||
run_migrations()
|
||||
init_restic()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,741 +0,0 @@
|
|||
"""
|
||||
This module contains the controller class for backups.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import time
|
||||
import os
|
||||
from os import statvfs
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from selfprivacy_api.services import (
|
||||
get_service_by_id,
|
||||
get_all_services,
|
||||
)
|
||||
from selfprivacy_api.services.service import (
|
||||
Service,
|
||||
ServiceStatus,
|
||||
StoppedService,
|
||||
)
|
||||
|
||||
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
||||
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
from selfprivacy_api.graphql.common_types.backup import (
|
||||
RestoreStrategy,
|
||||
BackupReason,
|
||||
AutobackupQuotas,
|
||||
)
|
||||
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
|
||||
from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.providers import get_provider
|
||||
from selfprivacy_api.backup.storage import Storage
|
||||
from selfprivacy_api.backup.jobs import (
|
||||
get_backup_job,
|
||||
get_backup_fail,
|
||||
add_backup_job,
|
||||
get_restore_job,
|
||||
add_restore_job,
|
||||
)
|
||||
|
||||
|
||||
BACKUP_PROVIDER_ENVS = {
|
||||
"kind": "BACKUP_KIND",
|
||||
"login": "BACKUP_LOGIN",
|
||||
"key": "BACKUP_KEY",
|
||||
"location": "BACKUP_LOCATION",
|
||||
}
|
||||
|
||||
AUTOBACKUP_JOB_EXPIRATION_SECONDS = 60 * 60 # one hour
|
||||
|
||||
|
||||
class NotDeadError(AssertionError):
|
||||
"""
|
||||
This error is raised when we try to back up a service that is not dead yet.
|
||||
"""
|
||||
|
||||
def __init__(self, service: Service):
|
||||
self.service_name = service.get_id()
|
||||
super().__init__()
|
||||
|
||||
def __str__(self):
|
||||
return f"""
|
||||
Service {self.service_name} should be either stopped or dead from
|
||||
an error before we back up.
|
||||
Normally, this error is unreachable because we do try ensure this.
|
||||
Apparently, not this time.
|
||||
"""
|
||||
|
||||
|
||||
class RotationBucket:
|
||||
"""
|
||||
Bucket object used for rotation.
|
||||
Has the following mutable fields:
|
||||
- the counter, int
|
||||
- the lambda function which takes datetime and the int and returns the int
|
||||
- the last, int
|
||||
"""
|
||||
|
||||
def __init__(self, counter: int, last: int, rotation_lambda):
|
||||
self.counter: int = counter
|
||||
self.last: int = last
|
||||
self.rotation_lambda: Callable[[datetime, int], int] = rotation_lambda
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"Bucket(counter={self.counter}, last={self.last})"
|
||||
|
||||
|
||||
class Backups:
|
||||
"""A stateless controller class for backups"""
|
||||
|
||||
# Providers
|
||||
|
||||
@staticmethod
|
||||
def provider() -> AbstractBackupProvider:
|
||||
"""
|
||||
Returns the current backup storage provider.
|
||||
"""
|
||||
return Backups._lookup_provider()
|
||||
|
||||
@staticmethod
|
||||
def set_provider(
|
||||
kind: BackupProviderEnum,
|
||||
login: str,
|
||||
key: str,
|
||||
location: str,
|
||||
repo_id: str = "",
|
||||
) -> None:
|
||||
"""
|
||||
Sets the new configuration of the backup storage provider.
|
||||
|
||||
In case of `BackupProviderEnum.BACKBLAZE`, the `login` is the key ID,
|
||||
the `key` is the key itself, and the `location` is the bucket name and
|
||||
the `repo_id` is the bucket ID.
|
||||
"""
|
||||
provider: AbstractBackupProvider = Backups._construct_provider(
|
||||
kind,
|
||||
login,
|
||||
key,
|
||||
location,
|
||||
repo_id,
|
||||
)
|
||||
Storage.store_provider(provider)
|
||||
|
||||
@staticmethod
|
||||
def reset() -> None:
|
||||
"""
|
||||
Deletes all the data about the backup storage provider.
|
||||
"""
|
||||
Storage.reset()
|
||||
|
||||
@staticmethod
|
||||
def _lookup_provider() -> AbstractBackupProvider:
|
||||
redis_provider = Backups._load_provider_redis()
|
||||
if redis_provider is not None:
|
||||
return redis_provider
|
||||
|
||||
none_provider = Backups._construct_provider(
|
||||
BackupProviderEnum.NONE, login="", key="", location=""
|
||||
)
|
||||
Storage.store_provider(none_provider)
|
||||
return none_provider
|
||||
|
||||
@staticmethod
|
||||
def set_provider_from_envs():
|
||||
for env in BACKUP_PROVIDER_ENVS.values():
|
||||
if env not in os.environ.keys():
|
||||
raise ValueError(
|
||||
f"Cannot set backup provider from envs, there is no {env} set"
|
||||
)
|
||||
|
||||
kind_str = os.environ[BACKUP_PROVIDER_ENVS["kind"]]
|
||||
kind_enum = BackupProviderEnum[kind_str]
|
||||
provider = Backups._construct_provider(
|
||||
kind=kind_enum,
|
||||
login=os.environ[BACKUP_PROVIDER_ENVS["login"]],
|
||||
key=os.environ[BACKUP_PROVIDER_ENVS["key"]],
|
||||
location=os.environ[BACKUP_PROVIDER_ENVS["location"]],
|
||||
)
|
||||
Storage.store_provider(provider)
|
||||
|
||||
@staticmethod
|
||||
def _construct_provider(
|
||||
kind: BackupProviderEnum,
|
||||
login: str,
|
||||
key: str,
|
||||
location: str,
|
||||
repo_id: str = "",
|
||||
) -> AbstractBackupProvider:
|
||||
provider_class = get_provider(kind)
|
||||
|
||||
return provider_class(
|
||||
login=login,
|
||||
key=key,
|
||||
location=location,
|
||||
repo_id=repo_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_provider_redis() -> Optional[AbstractBackupProvider]:
|
||||
provider_model = Storage.load_provider()
|
||||
if provider_model is None:
|
||||
return None
|
||||
return Backups._construct_provider(
|
||||
BackupProviderEnum[provider_model.kind],
|
||||
provider_model.login,
|
||||
provider_model.key,
|
||||
provider_model.location,
|
||||
provider_model.repo_id,
|
||||
)
|
||||
|
||||
# Init
|
||||
|
||||
@staticmethod
|
||||
def init_repo() -> None:
|
||||
"""
|
||||
Initializes the backup repository. This is required once per repo.
|
||||
"""
|
||||
Backups.provider().backupper.init()
|
||||
Storage.mark_as_init()
|
||||
|
||||
@staticmethod
|
||||
def erase_repo() -> None:
|
||||
"""
|
||||
Completely empties the remote
|
||||
"""
|
||||
Backups.provider().backupper.erase_repo()
|
||||
Storage.mark_as_uninitted()
|
||||
|
||||
@staticmethod
|
||||
def is_initted() -> bool:
|
||||
"""
|
||||
Returns whether the backup repository is initialized or not.
|
||||
If it is not initialized, we cannot back up and probably should
|
||||
call `init_repo` first.
|
||||
"""
|
||||
if Storage.has_init_mark():
|
||||
return True
|
||||
|
||||
initted = Backups.provider().backupper.is_initted()
|
||||
if initted:
|
||||
Storage.mark_as_init()
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# Backup
|
||||
|
||||
@staticmethod
|
||||
def back_up(
|
||||
service: Service, reason: BackupReason = BackupReason.EXPLICIT
|
||||
) -> Snapshot:
|
||||
"""The top-level function to back up a service
|
||||
If it fails for any reason at all, it should both mark job as
|
||||
errored and re-raise an error"""
|
||||
|
||||
job = get_backup_job(service)
|
||||
if job is None:
|
||||
job = add_backup_job(service)
|
||||
Jobs.update(job, status=JobStatus.RUNNING)
|
||||
|
||||
try:
|
||||
if service.can_be_backed_up() is False:
|
||||
raise ValueError("cannot backup a non-backuppable service")
|
||||
folders = service.get_folders()
|
||||
service_name = service.get_id()
|
||||
service.pre_backup()
|
||||
snapshot = Backups.provider().backupper.start_backup(
|
||||
folders,
|
||||
service_name,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
Backups._on_new_snapshot_created(service_name, snapshot)
|
||||
if reason == BackupReason.AUTO:
|
||||
Backups._prune_auto_snaps(service)
|
||||
service.post_restore()
|
||||
except Exception as error:
|
||||
Jobs.update(job, status=JobStatus.ERROR, error=str(error))
|
||||
raise error
|
||||
|
||||
Jobs.update(job, status=JobStatus.FINISHED)
|
||||
if reason in [BackupReason.AUTO, BackupReason.PRE_RESTORE]:
|
||||
Jobs.set_expiration(job, AUTOBACKUP_JOB_EXPIRATION_SECONDS)
|
||||
return Backups.sync_date_from_cache(snapshot)
|
||||
|
||||
@staticmethod
|
||||
def sync_date_from_cache(snapshot: Snapshot) -> Snapshot:
|
||||
"""
|
||||
Our snapshot creation dates are different from those on server by a tiny amount.
|
||||
This is a convenience, maybe it is better to write a special comparison
|
||||
function for snapshots
|
||||
"""
|
||||
return Storage.get_cached_snapshot_by_id(snapshot.id)
|
||||
|
||||
@staticmethod
|
||||
def _auto_snaps(service):
|
||||
return [
|
||||
snap
|
||||
for snap in Backups.get_snapshots(service)
|
||||
if snap.reason == BackupReason.AUTO
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _prune_snaps_with_quotas(snapshots: List[Snapshot]) -> List[Snapshot]:
|
||||
# Function broken out for testability
|
||||
# Sorting newest first
|
||||
sorted_snaps = sorted(snapshots, key=lambda s: s.created_at, reverse=True)
|
||||
quotas: AutobackupQuotas = Backups.autobackup_quotas()
|
||||
|
||||
buckets: list[RotationBucket] = [
|
||||
RotationBucket(
|
||||
quotas.last, # type: ignore
|
||||
-1,
|
||||
lambda _, index: index,
|
||||
),
|
||||
RotationBucket(
|
||||
quotas.daily, # type: ignore
|
||||
-1,
|
||||
lambda date, _: date.year * 10000 + date.month * 100 + date.day,
|
||||
),
|
||||
RotationBucket(
|
||||
quotas.weekly, # type: ignore
|
||||
-1,
|
||||
lambda date, _: date.year * 100 + date.isocalendar()[1],
|
||||
),
|
||||
RotationBucket(
|
||||
quotas.monthly, # type: ignore
|
||||
-1,
|
||||
lambda date, _: date.year * 100 + date.month,
|
||||
),
|
||||
RotationBucket(
|
||||
quotas.yearly, # type: ignore
|
||||
-1,
|
||||
lambda date, _: date.year,
|
||||
),
|
||||
]
|
||||
|
||||
new_snaplist: List[Snapshot] = []
|
||||
for i, snap in enumerate(sorted_snaps):
|
||||
keep_snap = False
|
||||
for bucket in buckets:
|
||||
if (bucket.counter > 0) or (bucket.counter == -1):
|
||||
val = bucket.rotation_lambda(snap.created_at, i)
|
||||
if (val != bucket.last) or (i == len(sorted_snaps) - 1):
|
||||
bucket.last = val
|
||||
if bucket.counter > 0:
|
||||
bucket.counter -= 1
|
||||
if not keep_snap:
|
||||
new_snaplist.append(snap)
|
||||
keep_snap = True
|
||||
|
||||
return new_snaplist
|
||||
|
||||
@staticmethod
|
||||
def _prune_auto_snaps(service) -> None:
|
||||
# Not very testable by itself, so most testing is going on Backups._prune_snaps_with_quotas
|
||||
# We can still test total limits and, say, daily limits
|
||||
|
||||
auto_snaps = Backups._auto_snaps(service)
|
||||
new_snaplist = Backups._prune_snaps_with_quotas(auto_snaps)
|
||||
|
||||
deletable_snaps = [snap for snap in auto_snaps if snap not in new_snaplist]
|
||||
Backups.forget_snapshots(deletable_snaps)
|
||||
|
||||
@staticmethod
|
||||
def _standardize_quotas(i: int) -> int:
|
||||
if i <= -1:
|
||||
i = -1
|
||||
return i
|
||||
|
||||
@staticmethod
|
||||
def autobackup_quotas() -> AutobackupQuotas:
|
||||
"""0 means do not keep, -1 means unlimited"""
|
||||
|
||||
return Storage.autobackup_quotas()
|
||||
|
||||
@staticmethod
|
||||
def set_autobackup_quotas(quotas: AutobackupQuotas) -> None:
|
||||
"""0 means do not keep, -1 means unlimited"""
|
||||
|
||||
Storage.set_autobackup_quotas(
|
||||
AutobackupQuotas(
|
||||
last=Backups._standardize_quotas(quotas.last), # type: ignore
|
||||
daily=Backups._standardize_quotas(quotas.daily), # type: ignore
|
||||
weekly=Backups._standardize_quotas(quotas.weekly), # type: ignore
|
||||
monthly=Backups._standardize_quotas(quotas.monthly), # type: ignore
|
||||
yearly=Backups._standardize_quotas(quotas.yearly), # type: ignore
|
||||
)
|
||||
)
|
||||
# do not prune all autosnaps right away, this will be done by an async task
|
||||
|
||||
@staticmethod
|
||||
def prune_all_autosnaps() -> None:
|
||||
for service in get_all_services():
|
||||
Backups._prune_auto_snaps(service)
|
||||
|
||||
# Restoring
|
||||
|
||||
@staticmethod
|
||||
def _ensure_queued_restore_job(service, snapshot) -> Job:
|
||||
job = get_restore_job(service)
|
||||
if job is None:
|
||||
job = add_restore_job(snapshot)
|
||||
|
||||
Jobs.update(job, status=JobStatus.CREATED)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def _inplace_restore(
|
||||
service: Service,
|
||||
snapshot: Snapshot,
|
||||
job: Job,
|
||||
) -> None:
|
||||
Jobs.update(
|
||||
job, status=JobStatus.CREATED, status_text="Waiting for pre-restore backup"
|
||||
)
|
||||
failsafe_snapshot = Backups.back_up(service, BackupReason.PRE_RESTORE)
|
||||
|
||||
Jobs.update(
|
||||
job, status=JobStatus.RUNNING, status_text=f"Restoring from {snapshot.id}"
|
||||
)
|
||||
try:
|
||||
Backups._restore_service_from_snapshot(
|
||||
service,
|
||||
snapshot.id,
|
||||
verify=False,
|
||||
)
|
||||
except Exception as error:
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
status_text=f"Restore failed with {str(error)}, reverting to {failsafe_snapshot.id}",
|
||||
)
|
||||
Backups._restore_service_from_snapshot(
|
||||
service, failsafe_snapshot.id, verify=False
|
||||
)
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
status_text=f"Restore failed with {str(error)}, reverted to {failsafe_snapshot.id}",
|
||||
)
|
||||
raise error
|
||||
|
||||
@staticmethod
|
||||
def restore_snapshot(
|
||||
snapshot: Snapshot, strategy=RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE
|
||||
) -> None:
|
||||
"""Restores a snapshot to its original service using the given strategy"""
|
||||
service = get_service_by_id(snapshot.service_name)
|
||||
if service is None:
|
||||
raise ValueError(
|
||||
f"snapshot has a nonexistent service: {snapshot.service_name}"
|
||||
)
|
||||
job = Backups._ensure_queued_restore_job(service, snapshot)
|
||||
|
||||
try:
|
||||
Backups._assert_restorable(snapshot)
|
||||
Jobs.update(
|
||||
job, status=JobStatus.RUNNING, status_text="Stopping the service"
|
||||
)
|
||||
with StoppedService(service):
|
||||
Backups.assert_dead(service)
|
||||
if strategy == RestoreStrategy.INPLACE:
|
||||
Backups._inplace_restore(service, snapshot, job)
|
||||
else: # verify_before_download is our default
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text=f"Restoring from {snapshot.id}",
|
||||
)
|
||||
Backups._restore_service_from_snapshot(
|
||||
service, snapshot.id, verify=True
|
||||
)
|
||||
|
||||
service.post_restore()
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.RUNNING,
|
||||
progress=90,
|
||||
status_text="Restarting the service",
|
||||
)
|
||||
|
||||
except Exception as error:
|
||||
Jobs.update(job, status=JobStatus.ERROR, status_text=str(error))
|
||||
raise error
|
||||
|
||||
Jobs.update(job, status=JobStatus.FINISHED)
|
||||
|
||||
@staticmethod
|
||||
def _assert_restorable(
|
||||
snapshot: Snapshot, strategy=RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE
|
||||
) -> None:
|
||||
service = get_service_by_id(snapshot.service_name)
|
||||
if service is None:
|
||||
raise ValueError(
|
||||
f"snapshot has a nonexistent service: {snapshot.service_name}"
|
||||
)
|
||||
|
||||
restored_snap_size = Backups.snapshot_restored_size(snapshot.id)
|
||||
|
||||
if strategy == RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE:
|
||||
needed_space = restored_snap_size
|
||||
elif strategy == RestoreStrategy.INPLACE:
|
||||
needed_space = restored_snap_size - service.get_storage_usage()
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"""
|
||||
We do not know if there is enough space for restoration because
|
||||
there is some novel restore strategy used!
|
||||
This is a developer's fault, open an issue please
|
||||
"""
|
||||
)
|
||||
available_space = Backups.space_usable_for_service(service)
|
||||
if needed_space > available_space:
|
||||
raise ValueError(
|
||||
f"we only have {available_space} bytes "
|
||||
f"but snapshot needs {needed_space}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _restore_service_from_snapshot(
|
||||
service: Service,
|
||||
snapshot_id: str,
|
||||
verify=True,
|
||||
) -> None:
|
||||
folders = service.get_folders()
|
||||
|
||||
Backups.provider().backupper.restore_from_backup(
|
||||
snapshot_id,
|
||||
folders,
|
||||
verify=verify,
|
||||
)
|
||||
|
||||
# Snapshots
|
||||
|
||||
@staticmethod
|
||||
def get_snapshots(service: Service) -> List[Snapshot]:
|
||||
"""Returns all snapshots for a given service"""
|
||||
snapshots = Backups.get_all_snapshots()
|
||||
service_id = service.get_id()
|
||||
return list(
|
||||
filter(
|
||||
lambda snap: snap.service_name == service_id,
|
||||
snapshots,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_all_snapshots() -> List[Snapshot]:
|
||||
"""Returns all snapshots"""
|
||||
# When we refresh our cache:
|
||||
# 1. Manually
|
||||
# 2. On timer
|
||||
# 3. On new snapshot
|
||||
# 4. On snapshot deletion
|
||||
|
||||
return Storage.get_cached_snapshots()
|
||||
|
||||
@staticmethod
|
||||
def get_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]:
|
||||
"""Returns a backup snapshot by its id"""
|
||||
snap = Storage.get_cached_snapshot_by_id(snapshot_id)
|
||||
if snap is not None:
|
||||
return snap
|
||||
|
||||
# Possibly our cache entry got invalidated, let's try one more time
|
||||
Backups.force_snapshot_cache_reload()
|
||||
snap = Storage.get_cached_snapshot_by_id(snapshot_id)
|
||||
|
||||
return snap
|
||||
|
||||
@staticmethod
|
||||
def forget_snapshots(snapshots: List[Snapshot]) -> None:
|
||||
"""
|
||||
Deletes a batch of snapshots from the repo and syncs cache
|
||||
Optimized
|
||||
"""
|
||||
ids = [snapshot.id for snapshot in snapshots]
|
||||
Backups.provider().backupper.forget_snapshots(ids)
|
||||
|
||||
Backups.force_snapshot_cache_reload()
|
||||
|
||||
@staticmethod
|
||||
def forget_snapshot(snapshot: Snapshot) -> None:
|
||||
"""Deletes a snapshot from the repo and from cache"""
|
||||
Backups.forget_snapshots([snapshot])
|
||||
|
||||
@staticmethod
|
||||
def forget_all_snapshots():
|
||||
"""
|
||||
Mark all snapshots we have made for deletion and make them inaccessible
|
||||
(this is done by cloud, we only issue a command)
|
||||
"""
|
||||
Backups.forget_snapshots(Backups.get_all_snapshots())
|
||||
|
||||
@staticmethod
|
||||
def force_snapshot_cache_reload() -> None:
|
||||
"""
|
||||
Forces a reload of the snapshot cache.
|
||||
|
||||
This may be an expensive operation, so use it wisely.
|
||||
User pays for the API calls.
|
||||
"""
|
||||
upstream_snapshots = Backups.provider().backupper.get_snapshots()
|
||||
Storage.invalidate_snapshot_storage()
|
||||
for snapshot in upstream_snapshots:
|
||||
Storage.cache_snapshot(snapshot)
|
||||
|
||||
@staticmethod
|
||||
def snapshot_restored_size(snapshot_id: str) -> int:
|
||||
"""Returns the size of the snapshot"""
|
||||
return Backups.provider().backupper.restored_size(
|
||||
snapshot_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _on_new_snapshot_created(service_id: str, snapshot: Snapshot) -> None:
|
||||
"""What do we do with a snapshot that is just made?"""
|
||||
# non-expiring timestamp of the last
|
||||
Storage.store_last_timestamp(service_id, snapshot)
|
||||
Backups.force_snapshot_cache_reload()
|
||||
|
||||
# Autobackup
|
||||
|
||||
@staticmethod
|
||||
def autobackup_period_minutes() -> Optional[int]:
|
||||
"""None means autobackup is disabled"""
|
||||
return Storage.autobackup_period_minutes()
|
||||
|
||||
@staticmethod
|
||||
def set_autobackup_period_minutes(minutes: int) -> None:
|
||||
"""
|
||||
0 and negative numbers are equivalent to disable.
|
||||
Setting to a positive number may result in a backup very soon
|
||||
if some services are not backed up.
|
||||
"""
|
||||
if minutes <= 0:
|
||||
Backups.disable_all_autobackup()
|
||||
return
|
||||
Storage.store_autobackup_period_minutes(minutes)
|
||||
|
||||
@staticmethod
|
||||
def disable_all_autobackup() -> None:
|
||||
"""
|
||||
Disables all automatic backing up,
|
||||
but does not change per-service settings
|
||||
"""
|
||||
Storage.delete_backup_period()
|
||||
|
||||
@staticmethod
|
||||
def is_time_to_backup(time: datetime) -> bool:
|
||||
"""
|
||||
Intended as a time validator for huey cron scheduler
|
||||
of automatic backups
|
||||
"""
|
||||
|
||||
return Backups.services_to_back_up(time) != []
|
||||
|
||||
@staticmethod
|
||||
def services_to_back_up(time: datetime) -> List[Service]:
|
||||
"""Returns a list of services that should be backed up at a given time"""
|
||||
return [
|
||||
service
|
||||
for service in get_all_services()
|
||||
if Backups.is_time_to_backup_service(service, time)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_last_backed_up(service: Service) -> Optional[datetime]:
|
||||
"""Get a timezone-aware time of the last backup of a service"""
|
||||
return Storage.get_last_backup_time(service.get_id())
|
||||
|
||||
@staticmethod
|
||||
def get_last_backup_error_time(service: Service) -> Optional[datetime]:
|
||||
"""Get a timezone-aware time of the last backup of a service"""
|
||||
job = get_backup_fail(service)
|
||||
if job is not None:
|
||||
datetime_created = job.created_at
|
||||
if datetime_created.tzinfo is None:
|
||||
# assume it is in localtime
|
||||
offset = timedelta(seconds=time.localtime().tm_gmtoff)
|
||||
datetime_created = datetime_created - offset
|
||||
return datetime.combine(
|
||||
datetime_created.date(), datetime_created.time(), timezone.utc
|
||||
)
|
||||
return datetime_created
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_time_to_backup_service(service: Service, time: datetime):
|
||||
"""Returns True if it is time to back up a service"""
|
||||
period = Backups.autobackup_period_minutes()
|
||||
if period is None:
|
||||
return False
|
||||
|
||||
if not service.is_enabled():
|
||||
return False
|
||||
if not service.can_be_backed_up():
|
||||
return False
|
||||
|
||||
last_error = Backups.get_last_backup_error_time(service)
|
||||
|
||||
if last_error is not None:
|
||||
if time < last_error + timedelta(seconds=AUTOBACKUP_JOB_EXPIRATION_SECONDS):
|
||||
return False
|
||||
|
||||
last_backup = Backups.get_last_backed_up(service)
|
||||
|
||||
# Queue a backup immediately if there are no previous backups
|
||||
if last_backup is None:
|
||||
return True
|
||||
|
||||
if time > last_backup + timedelta(minutes=period):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# Helpers
|
||||
|
||||
@staticmethod
|
||||
def space_usable_for_service(service: Service) -> int:
|
||||
"""
|
||||
Returns the amount of space available on the volume the given
|
||||
service is located on.
|
||||
"""
|
||||
folders = service.get_folders()
|
||||
if folders == []:
|
||||
raise ValueError("unallocated service", service.get_id())
|
||||
|
||||
# We assume all folders of one service live at the same volume
|
||||
fs_info = statvfs(folders[0])
|
||||
usable_bytes = fs_info.f_frsize * fs_info.f_bavail
|
||||
return usable_bytes
|
||||
|
||||
@staticmethod
|
||||
def set_localfile_repo(file_path: str):
|
||||
"""Used by tests to set a local folder as a backup repo"""
|
||||
# pylint: disable-next=invalid-name
|
||||
ProviderClass = get_provider(BackupProviderEnum.FILE)
|
||||
provider = ProviderClass(
|
||||
login="",
|
||||
key="",
|
||||
location=file_path,
|
||||
repo_id="",
|
||||
)
|
||||
Storage.store_provider(provider)
|
||||
|
||||
@staticmethod
|
||||
def assert_dead(service: Service):
|
||||
"""
|
||||
Checks if a service is dead and can be safely restored from a snapshot.
|
||||
"""
|
||||
if service.get_status() not in [
|
||||
ServiceStatus.INACTIVE,
|
||||
ServiceStatus.FAILED,
|
||||
]:
|
||||
raise NotDeadError(service)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from typing import List
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.graphql.common_types.backup import BackupReason
|
||||
|
||||
|
||||
class AbstractBackupper(ABC):
|
||||
"""Abstract class for backuppers"""
|
||||
|
||||
# flake8: noqa: B027
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def is_initted(self) -> bool:
|
||||
"""Returns true if the repository is initted"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def set_creds(self, account: str, key: str, repo: str) -> None:
|
||||
"""Set the credentials for the backupper"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def start_backup(
|
||||
self,
|
||||
folders: List[str],
|
||||
service_name: str,
|
||||
reason: BackupReason = BackupReason.EXPLICIT,
|
||||
) -> Snapshot:
|
||||
"""Start a backup of the given folders"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_snapshots(self) -> List[Snapshot]:
|
||||
"""Get all snapshots from the repo"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def init(self) -> None:
|
||||
"""Initialize the repository"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def erase_repo(self) -> None:
|
||||
"""Completely empties the remote"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def restore_from_backup(
|
||||
self,
|
||||
snapshot_id: str,
|
||||
folders: List[str],
|
||||
verify=True,
|
||||
) -> None:
|
||||
"""Restore a target folder using a snapshot"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def restored_size(self, snapshot_id: str) -> int:
|
||||
"""Get the size of the restored snapshot"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forget_snapshot(self, snapshot_id) -> None:
|
||||
"""Forget a snapshot"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forget_snapshots(self, snapshot_ids: List[str]) -> None:
|
||||
"""Maybe optimized deletion of a batch of snapshots, just cycling if unsupported"""
|
||||
raise NotImplementedError
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
from typing import List
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||
from selfprivacy_api.graphql.common_types.backup import BackupReason
|
||||
|
||||
|
||||
class NoneBackupper(AbstractBackupper):
|
||||
"""A backupper that does nothing"""
|
||||
|
||||
def is_initted(self, repo_name: str = "") -> bool:
|
||||
return False
|
||||
|
||||
def set_creds(self, account: str, key: str, repo: str):
|
||||
pass
|
||||
|
||||
def start_backup(
|
||||
self, folders: List[str], tag: str, reason: BackupReason = BackupReason.EXPLICIT
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_snapshots(self) -> List[Snapshot]:
|
||||
"""Get all snapshots from the repo"""
|
||||
return []
|
||||
|
||||
def init(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def erase_repo(self) -> None:
|
||||
"""Completely empties the remote"""
|
||||
# this one is already empty
|
||||
pass
|
||||
|
||||
def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True):
|
||||
"""Restore a target folder using a snapshot"""
|
||||
raise NotImplementedError
|
||||
|
||||
def restored_size(self, snapshot_id: str) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
def forget_snapshot(self, snapshot_id):
|
||||
raise NotImplementedError("forget_snapshot")
|
||||
|
||||
def forget_snapshots(self, snapshots):
|
||||
raise NotImplementedError("forget_snapshots")
|
||||
|
|
@ -1,554 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import json
|
||||
import datetime
|
||||
import tempfile
|
||||
|
||||
from typing import List, Optional, TypeVar, Callable
|
||||
from collections.abc import Iterable
|
||||
from json.decoder import JSONDecodeError
|
||||
from os.path import exists, join
|
||||
from os import mkdir
|
||||
from shutil import rmtree
|
||||
|
||||
from selfprivacy_api.graphql.common_types.backup import BackupReason
|
||||
from selfprivacy_api.backup.util import output_yielder, sync
|
||||
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.backup.jobs import get_backup_job
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
||||
|
||||
from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
||||
|
||||
SHORT_ID_LEN = 8
|
||||
|
||||
T = TypeVar("T", bound=Callable)
|
||||
|
||||
|
||||
def unlocked_repo(func: T) -> T:
|
||||
"""unlock repo and retry if it appears to be locked"""
|
||||
|
||||
def inner(self: ResticBackupper, *args, **kwargs):
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except Exception as error:
|
||||
if "unable to create lock" in str(error):
|
||||
self.unlock()
|
||||
return func(self, *args, **kwargs)
|
||||
else:
|
||||
raise error
|
||||
|
||||
# Above, we manually guarantee that the type returned is compatible.
|
||||
return inner # type: ignore
|
||||
|
||||
|
||||
class ResticBackupper(AbstractBackupper):
|
||||
def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None:
|
||||
self.login_flag = login_flag
|
||||
self.key_flag = key_flag
|
||||
self.storage_type = storage_type
|
||||
self.account = ""
|
||||
self.key = ""
|
||||
self.repo = ""
|
||||
super().__init__()
|
||||
|
||||
def set_creds(self, account: str, key: str, repo: str) -> None:
|
||||
self.account = account
|
||||
self.key = key
|
||||
self.repo = repo
|
||||
|
||||
def restic_repo(self) -> str:
|
||||
# https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone
|
||||
# https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5
|
||||
return f"rclone:{self.rclone_repo()}"
|
||||
|
||||
def rclone_repo(self) -> str:
|
||||
return f"{self.storage_type}{self.repo}"
|
||||
|
||||
def rclone_args(self):
|
||||
return "rclone.args=serve restic --stdio " + " ".join(
|
||||
self.backend_rclone_args()
|
||||
)
|
||||
|
||||
def backend_rclone_args(self) -> list[str]:
|
||||
args = []
|
||||
if self.account != "":
|
||||
acc_args = [self.login_flag, self.account]
|
||||
args.extend(acc_args)
|
||||
if self.key != "":
|
||||
key_args = [self.key_flag, self.key]
|
||||
args.extend(key_args)
|
||||
return args
|
||||
|
||||
def _password_command(self):
|
||||
return f"echo {LocalBackupSecret.get()}"
|
||||
|
||||
def restic_command(self, *args, tags: Optional[List[str]] = None) -> List[str]:
|
||||
"""
|
||||
Construct a restic command against the currently configured repo
|
||||
Can support [nested] arrays as arguments, will flatten them into the final commmand
|
||||
"""
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
command = [
|
||||
"restic",
|
||||
"-o",
|
||||
self.rclone_args(),
|
||||
"-r",
|
||||
self.restic_repo(),
|
||||
"--password-command",
|
||||
self._password_command(),
|
||||
]
|
||||
if tags != []:
|
||||
for tag in tags:
|
||||
command.extend(
|
||||
[
|
||||
"--tag",
|
||||
tag,
|
||||
]
|
||||
)
|
||||
if args:
|
||||
command.extend(ResticBackupper.__flatten_list(args))
|
||||
return command
|
||||
|
||||
def erase_repo(self) -> None:
|
||||
"""Fully erases repo on remote, can be reinitted again"""
|
||||
command = [
|
||||
"rclone",
|
||||
"purge",
|
||||
self.rclone_repo(),
|
||||
]
|
||||
backend_args = self.backend_rclone_args()
|
||||
if backend_args:
|
||||
command.extend(backend_args)
|
||||
|
||||
with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle:
|
||||
output = handle.communicate()[0].decode("utf-8")
|
||||
if handle.returncode != 0:
|
||||
raise ValueError(
|
||||
"purge exited with errorcode",
|
||||
handle.returncode,
|
||||
":",
|
||||
output,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __flatten_list(list_to_flatten):
|
||||
"""string-aware list flattener"""
|
||||
result = []
|
||||
for item in list_to_flatten:
|
||||
if isinstance(item, Iterable) and not isinstance(item, str):
|
||||
result.extend(ResticBackupper.__flatten_list(item))
|
||||
continue
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _run_backup_command(
|
||||
backup_command: List[str], job: Optional[Job]
|
||||
) -> List[dict]:
|
||||
"""And handle backup output"""
|
||||
messages = []
|
||||
output = []
|
||||
restic_reported_error = False
|
||||
|
||||
for raw_message in output_yielder(backup_command):
|
||||
if "ERROR:" in raw_message:
|
||||
restic_reported_error = True
|
||||
output.append(raw_message)
|
||||
|
||||
if not restic_reported_error:
|
||||
message = ResticBackupper.parse_message(raw_message, job)
|
||||
messages.append(message)
|
||||
|
||||
if restic_reported_error:
|
||||
raise ValueError(
|
||||
"Restic returned error(s): ",
|
||||
output,
|
||||
)
|
||||
|
||||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _replace_in_array(array: List[str], target, replacement) -> None:
|
||||
if target == "":
|
||||
return
|
||||
|
||||
for i, value in enumerate(array):
|
||||
if target in value:
|
||||
array[i] = array[i].replace(target, replacement)
|
||||
|
||||
def _censor_command(self, command: List[str]) -> List[str]:
|
||||
result = command.copy()
|
||||
ResticBackupper._replace_in_array(result, self.key, "CENSORED")
|
||||
ResticBackupper._replace_in_array(result, LocalBackupSecret.get(), "CENSORED")
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _get_backup_job(service_name: str) -> Optional[Job]:
|
||||
service = get_service_by_id(service_name)
|
||||
if service is None:
|
||||
raise ValueError("No service with id ", service_name)
|
||||
|
||||
return get_backup_job(service)
|
||||
|
||||
@unlocked_repo
|
||||
def start_backup(
|
||||
self,
|
||||
folders: List[str],
|
||||
service_name: str,
|
||||
reason: BackupReason = BackupReason.EXPLICIT,
|
||||
) -> Snapshot:
|
||||
"""
|
||||
Start backup with restic
|
||||
"""
|
||||
assert len(folders) != 0
|
||||
|
||||
job = ResticBackupper._get_backup_job(service_name)
|
||||
|
||||
tags = [service_name, reason.value]
|
||||
backup_command = self.restic_command(
|
||||
"backup",
|
||||
"--json",
|
||||
folders,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
try:
|
||||
messages = ResticBackupper._run_backup_command(backup_command, job)
|
||||
|
||||
id = ResticBackupper._snapshot_id_from_backup_messages(messages)
|
||||
return Snapshot(
|
||||
created_at=datetime.datetime.now(datetime.timezone.utc),
|
||||
id=id,
|
||||
service_name=service_name,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
"Could not create a snapshot: ",
|
||||
str(error),
|
||||
"command: ",
|
||||
self._censor_command(backup_command),
|
||||
) from error
|
||||
|
||||
@staticmethod
|
||||
def _snapshot_id_from_backup_messages(messages) -> str:
|
||||
for message in messages:
|
||||
if message["message_type"] == "summary":
|
||||
# There is a discrepancy between versions of restic/rclone
|
||||
# Some report short_id in this field and some full
|
||||
return message["snapshot_id"][0:SHORT_ID_LEN]
|
||||
|
||||
raise ValueError("no summary message in restic json output")
|
||||
|
||||
@staticmethod
|
||||
def parse_message(raw_message_line: str, job: Optional[Job] = None) -> dict:
|
||||
message = ResticBackupper.parse_json_output(raw_message_line)
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError("we have too many messages on one line?")
|
||||
if message["message_type"] == "status":
|
||||
if job is not None: # only update status if we run under some job
|
||||
Jobs.update(
|
||||
job,
|
||||
JobStatus.RUNNING,
|
||||
progress=int(message["percent_done"] * 100),
|
||||
)
|
||||
return message
|
||||
|
||||
def init(self) -> None:
|
||||
init_command = self.restic_command(
|
||||
"init",
|
||||
)
|
||||
with subprocess.Popen(
|
||||
init_command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as process_handle:
|
||||
output = process_handle.communicate()[0].decode("utf-8")
|
||||
if "created restic repository" not in output:
|
||||
raise ValueError("cannot init a repo: " + output)
|
||||
|
||||
@unlocked_repo
|
||||
def is_initted(self) -> bool:
|
||||
command = self.restic_command(
|
||||
"check",
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
shell=False,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as handle:
|
||||
output = handle.communicate()[0].decode("utf-8")
|
||||
if handle.returncode != 0:
|
||||
if "unable to create lock" in output:
|
||||
raise ValueError("Stale lock detected: ", output)
|
||||
return False
|
||||
return True
|
||||
|
||||
def unlock(self) -> None:
|
||||
"""Remove stale locks."""
|
||||
command = self.restic_command(
|
||||
"unlock",
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
shell=False,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as handle:
|
||||
# communication forces to complete and for returncode to get defined
|
||||
output = handle.communicate()[0].decode("utf-8")
|
||||
if handle.returncode != 0:
|
||||
raise ValueError("cannot unlock the backup repository: ", output)
|
||||
|
||||
def lock(self) -> None:
|
||||
"""
|
||||
Introduce a stale lock.
|
||||
Mainly for testing purposes.
|
||||
Double lock is supposed to fail
|
||||
"""
|
||||
command = self.restic_command(
|
||||
"check",
|
||||
)
|
||||
|
||||
# using temporary cache in /run/user/1000/restic-check-cache-817079729
|
||||
# repository 9639c714 opened (repository version 2) successfully, password is correct
|
||||
# created new cache in /run/user/1000/restic-check-cache-817079729
|
||||
# create exclusive lock for repository
|
||||
# load indexes
|
||||
# check all packs
|
||||
# check snapshots, trees and blobs
|
||||
# [0:00] 100.00% 1 / 1 snapshots
|
||||
# no errors were found
|
||||
|
||||
try:
|
||||
for line in output_yielder(command):
|
||||
if "indexes" in line:
|
||||
break
|
||||
if "unable" in line:
|
||||
raise ValueError(line)
|
||||
except Exception as error:
|
||||
raise ValueError("could not lock repository") from error
|
||||
|
||||
@unlocked_repo
|
||||
def restored_size(self, snapshot_id: str) -> int:
|
||||
"""
|
||||
Size of a snapshot
|
||||
"""
|
||||
command = self.restic_command(
|
||||
"stats",
|
||||
snapshot_id,
|
||||
"--json",
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
shell=False,
|
||||
) as handle:
|
||||
output = handle.communicate()[0].decode("utf-8")
|
||||
try:
|
||||
parsed_output = ResticBackupper.parse_json_output(output)
|
||||
return parsed_output["total_size"]
|
||||
except ValueError as error:
|
||||
raise ValueError("cannot restore a snapshot: " + output) from error
|
||||
|
||||
@unlocked_repo
|
||||
def restore_from_backup(
|
||||
self,
|
||||
snapshot_id,
|
||||
folders: List[str],
|
||||
verify=True,
|
||||
) -> None:
|
||||
"""
|
||||
Restore from backup with restic
|
||||
"""
|
||||
if folders is None or folders == []:
|
||||
raise ValueError("cannot restore without knowing where to!")
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
if verify:
|
||||
self._raw_verified_restore(snapshot_id, target=temp_dir)
|
||||
snapshot_root = temp_dir
|
||||
for folder in folders:
|
||||
src = join(snapshot_root, folder.strip("/"))
|
||||
if not exists(src):
|
||||
raise ValueError(
|
||||
f"No such path: {src}. We tried to find {folder}"
|
||||
)
|
||||
dst = folder
|
||||
sync(src, dst)
|
||||
|
||||
else: # attempting inplace restore
|
||||
for folder in folders:
|
||||
rmtree(folder)
|
||||
mkdir(folder)
|
||||
self._raw_verified_restore(snapshot_id, target="/")
|
||||
return
|
||||
|
||||
def _raw_verified_restore(self, snapshot_id, target="/"):
|
||||
"""barebones restic restore"""
|
||||
restore_command = self.restic_command(
|
||||
"restore", snapshot_id, "--target", target, "--verify"
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
restore_command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
shell=False,
|
||||
) as handle:
|
||||
# for some reason restore does not support
|
||||
# nice reporting of progress via json
|
||||
output = handle.communicate()[0].decode("utf-8")
|
||||
if "restoring" not in output:
|
||||
raise ValueError("cannot restore a snapshot: " + output)
|
||||
|
||||
assert (
|
||||
handle.returncode is not None
|
||||
) # none should be impossible after communicate
|
||||
if handle.returncode != 0:
|
||||
raise ValueError(
|
||||
"restore exited with errorcode",
|
||||
handle.returncode,
|
||||
":",
|
||||
output,
|
||||
)
|
||||
|
||||
def forget_snapshot(self, snapshot_id: str) -> None:
|
||||
self.forget_snapshots([snapshot_id])
|
||||
|
||||
@unlocked_repo
|
||||
def forget_snapshots(self, snapshot_ids: List[str]) -> None:
|
||||
# in case the backupper program supports batching, otherwise implement it by cycling
|
||||
forget_command = self.restic_command(
|
||||
"forget",
|
||||
[snapshot_ids],
|
||||
# TODO: prune should be done in a separate process
|
||||
"--prune",
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
forget_command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
shell=False,
|
||||
) as handle:
|
||||
# for some reason restore does not support
|
||||
# nice reporting of progress via json
|
||||
output, err = [
|
||||
string.decode(
|
||||
"utf-8",
|
||||
)
|
||||
for string in handle.communicate()
|
||||
]
|
||||
|
||||
if "no matching ID found" in err:
|
||||
raise ValueError(
|
||||
"trying to delete, but no such snapshot(s): ", snapshot_ids
|
||||
)
|
||||
|
||||
assert (
|
||||
handle.returncode is not None
|
||||
) # none should be impossible after communicate
|
||||
if handle.returncode != 0:
|
||||
raise ValueError(
|
||||
"forget exited with errorcode", handle.returncode, ":", output, err
|
||||
)
|
||||
|
||||
def _load_snapshots(self) -> object:
|
||||
"""
|
||||
Load list of snapshots from repository
|
||||
raises Value Error if repo does not exist
|
||||
"""
|
||||
listing_command = self.restic_command(
|
||||
"snapshots",
|
||||
"--json",
|
||||
)
|
||||
|
||||
with subprocess.Popen(
|
||||
listing_command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as backup_listing_process_descriptor:
|
||||
output = backup_listing_process_descriptor.communicate()[0].decode("utf-8")
|
||||
|
||||
if "Is there a repository at the following location?" in output:
|
||||
raise ValueError("No repository! : " + output)
|
||||
try:
|
||||
return ResticBackupper.parse_json_output(output)
|
||||
except ValueError as error:
|
||||
raise ValueError("Cannot load snapshots: ", output) from error
|
||||
|
||||
@unlocked_repo
|
||||
def get_snapshots(self) -> List[Snapshot]:
|
||||
"""Get all snapshots from the repo"""
|
||||
snapshots = []
|
||||
|
||||
for restic_snapshot in self._load_snapshots():
|
||||
# Compatibility with previous snaps:
|
||||
if len(restic_snapshot["tags"]) == 1:
|
||||
reason = BackupReason.EXPLICIT
|
||||
else:
|
||||
reason = restic_snapshot["tags"][1]
|
||||
|
||||
snapshot = Snapshot(
|
||||
id=restic_snapshot["short_id"],
|
||||
created_at=restic_snapshot["time"],
|
||||
service_name=restic_snapshot["tags"][0],
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
snapshots.append(snapshot)
|
||||
return snapshots
|
||||
|
||||
@staticmethod
|
||||
def parse_json_output(output: str) -> object:
|
||||
starting_index = ResticBackupper.json_start(output)
|
||||
|
||||
if starting_index == -1:
|
||||
raise ValueError("There is no json in the restic output: " + output)
|
||||
|
||||
truncated_output = output[starting_index:]
|
||||
json_messages = truncated_output.splitlines()
|
||||
if len(json_messages) == 1:
|
||||
try:
|
||||
return json.loads(truncated_output)
|
||||
except JSONDecodeError as error:
|
||||
raise ValueError(
|
||||
"There is no json in the restic output : " + output
|
||||
) from error
|
||||
|
||||
result_array = []
|
||||
for message in json_messages:
|
||||
result_array.append(json.loads(message))
|
||||
return result_array
|
||||
|
||||
@staticmethod
|
||||
def json_start(output: str) -> int:
|
||||
indices = [
|
||||
output.find("["),
|
||||
output.find("{"),
|
||||
]
|
||||
indices = [x for x in indices if x != -1]
|
||||
|
||||
if indices == []:
|
||||
return -1
|
||||
return min(indices)
|
||||
|
||||
@staticmethod
|
||||
def has_json(output: str) -> bool:
|
||||
if ResticBackupper.json_start(output) == -1:
|
||||
return False
|
||||
return True
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
from typing import Optional, List
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.jobs import Jobs, Job, JobStatus
|
||||
from selfprivacy_api.services.service import Service
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
|
||||
|
||||
def job_type_prefix(service: Service) -> str:
|
||||
return f"services.{service.get_id()}"
|
||||
|
||||
|
||||
def backup_job_type(service: Service) -> str:
|
||||
return f"{job_type_prefix(service)}.backup"
|
||||
|
||||
|
||||
def autobackup_job_type() -> str:
|
||||
return "backups.autobackup"
|
||||
|
||||
|
||||
def restore_job_type(service: Service) -> str:
|
||||
return f"{job_type_prefix(service)}.restore"
|
||||
|
||||
|
||||
def get_jobs_by_service(service: Service) -> List[Job]:
|
||||
result = []
|
||||
for job in Jobs.get_jobs():
|
||||
if job.type_id.startswith(job_type_prefix(service)) and job.status in [
|
||||
JobStatus.CREATED,
|
||||
JobStatus.RUNNING,
|
||||
]:
|
||||
result.append(job)
|
||||
return result
|
||||
|
||||
|
||||
def is_something_running_for(service: Service) -> bool:
|
||||
running_jobs = [
|
||||
job for job in get_jobs_by_service(service) if job.status == JobStatus.RUNNING
|
||||
]
|
||||
return len(running_jobs) != 0
|
||||
|
||||
|
||||
def add_autobackup_job(services: List[Service]) -> Job:
|
||||
service_names = [s.get_display_name() for s in services]
|
||||
pretty_service_list: str = ", ".join(service_names)
|
||||
job = Jobs.add(
|
||||
type_id=autobackup_job_type(),
|
||||
name="Automatic backup",
|
||||
description=f"Scheduled backup for services: {pretty_service_list}",
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
def add_backup_job(service: Service) -> Job:
|
||||
if is_something_running_for(service):
|
||||
message = (
|
||||
f"Cannot start a backup of {service.get_id()}, another operation is running: "
|
||||
+ get_jobs_by_service(service)[0].type_id
|
||||
)
|
||||
raise ValueError(message)
|
||||
display_name = service.get_display_name()
|
||||
job = Jobs.add(
|
||||
type_id=backup_job_type(service),
|
||||
name=f"Backup {display_name}",
|
||||
description=f"Backing up {display_name}",
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
def add_restore_job(snapshot: Snapshot) -> Job:
|
||||
service = get_service_by_id(snapshot.service_name)
|
||||
if service is None:
|
||||
raise ValueError(f"no such service: {snapshot.service_name}")
|
||||
if is_something_running_for(service):
|
||||
message = (
|
||||
f"Cannot start a restore of {service.get_id()}, another operation is running: "
|
||||
+ get_jobs_by_service(service)[0].type_id
|
||||
)
|
||||
raise ValueError(message)
|
||||
display_name = service.get_display_name()
|
||||
job = Jobs.add(
|
||||
type_id=restore_job_type(service),
|
||||
name=f"Restore {display_name}",
|
||||
description=f"restoring {display_name} from {snapshot.id}",
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
def get_job_by_type(type_id: str) -> Optional[Job]:
|
||||
for job in Jobs.get_jobs():
|
||||
if job.type_id == type_id and job.status in [
|
||||
JobStatus.CREATED,
|
||||
JobStatus.RUNNING,
|
||||
]:
|
||||
return job
|
||||
return None
|
||||
|
||||
|
||||
def get_failed_job_by_type(type_id: str) -> Optional[Job]:
|
||||
for job in Jobs.get_jobs():
|
||||
if job.type_id == type_id and job.status == JobStatus.ERROR:
|
||||
return job
|
||||
return None
|
||||
|
||||
|
||||
def get_backup_job(service: Service) -> Optional[Job]:
|
||||
return get_job_by_type(backup_job_type(service))
|
||||
|
||||
|
||||
def get_backup_fail(service: Service) -> Optional[Job]:
|
||||
return get_failed_job_by_type(backup_job_type(service))
|
||||
|
||||
|
||||
def get_restore_job(service: Service) -> Optional[Job]:
|
||||
return get_job_by_type(restore_job_type(service))
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
"""Handling of local secret used for encrypted backups.
|
||||
Separated out for circular dependency reasons
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import secrets
|
||||
|
||||
from selfprivacy_api.utils.redis_pool import RedisPool
|
||||
|
||||
|
||||
REDIS_KEY = "backup:local_secret"
|
||||
|
||||
redis = RedisPool().get_connection()
|
||||
|
||||
|
||||
class LocalBackupSecret:
|
||||
@staticmethod
|
||||
def get() -> str:
|
||||
"""A secret string which backblaze/other clouds do not know.
|
||||
Serves as encryption key.
|
||||
"""
|
||||
if not LocalBackupSecret.exists():
|
||||
LocalBackupSecret.reset()
|
||||
return redis.get(REDIS_KEY) # type: ignore
|
||||
|
||||
@staticmethod
|
||||
def set(secret: str):
|
||||
redis.set(REDIS_KEY, secret)
|
||||
|
||||
@staticmethod
|
||||
def reset():
|
||||
new_secret = LocalBackupSecret._generate()
|
||||
LocalBackupSecret.set(new_secret)
|
||||
|
||||
@staticmethod
|
||||
def _full_reset():
|
||||
redis.delete(REDIS_KEY)
|
||||
|
||||
@staticmethod
|
||||
def exists() -> bool:
|
||||
return redis.exists(REDIS_KEY) == 1
|
||||
|
||||
@staticmethod
|
||||
def _generate() -> str:
|
||||
return secrets.token_urlsafe(256)
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
from typing import Type
|
||||
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
|
||||
|
||||
from selfprivacy_api.backup.providers.backblaze import Backblaze
|
||||
from selfprivacy_api.backup.providers.memory import InMemoryBackup
|
||||
from selfprivacy_api.backup.providers.local_file import LocalFileBackup
|
||||
from selfprivacy_api.backup.providers.none import NoBackups
|
||||
|
||||
PROVIDER_MAPPING: dict[BackupProviderEnum, Type[AbstractBackupProvider]] = {
|
||||
BackupProviderEnum.BACKBLAZE: Backblaze,
|
||||
BackupProviderEnum.MEMORY: InMemoryBackup,
|
||||
BackupProviderEnum.FILE: LocalFileBackup,
|
||||
BackupProviderEnum.NONE: NoBackups,
|
||||
}
|
||||
|
||||
|
||||
def get_provider(
|
||||
provider_type: BackupProviderEnum,
|
||||
) -> Type[AbstractBackupProvider]:
|
||||
if provider_type not in PROVIDER_MAPPING.keys():
|
||||
raise LookupError("could not look up provider", provider_type)
|
||||
return PROVIDER_MAPPING[provider_type]
|
||||
|
||||
|
||||
def get_kind(provider: AbstractBackupProvider) -> str:
|
||||
"""Get the kind of the provider in the form of a string"""
|
||||
return provider.name.value
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from .provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
|
||||
|
||||
class Backblaze(AbstractBackupProvider):
|
||||
backupper = ResticBackupper("--b2-account", "--b2-key", ":b2:")
|
||||
|
||||
name = BackupProviderEnum.BACKBLAZE
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from .provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
|
||||
|
||||
class LocalFileBackup(AbstractBackupProvider):
|
||||
backupper = ResticBackupper("", "", ":local:")
|
||||
|
||||
name = BackupProviderEnum.FILE
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from .provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
|
||||
|
||||
class InMemoryBackup(AbstractBackupProvider):
|
||||
backupper = ResticBackupper("", "", ":memory:")
|
||||
|
||||
name = BackupProviderEnum.MEMORY
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.backuppers.none_backupper import NoneBackupper
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
|
||||
|
||||
class NoBackups(AbstractBackupProvider):
|
||||
backupper = NoneBackupper()
|
||||
|
||||
name = BackupProviderEnum.NONE
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
"""
|
||||
An abstract class for BackBlaze, S3 etc.
|
||||
It assumes that while some providers are supported via restic/rclone, others
|
||||
may require different backends
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||
from selfprivacy_api.graphql.queries.providers import (
|
||||
BackupProvider as BackupProviderEnum,
|
||||
)
|
||||
|
||||
|
||||
class AbstractBackupProvider(ABC):
|
||||
backupper: AbstractBackupper
|
||||
|
||||
name: BackupProviderEnum
|
||||
|
||||
def __init__(self, login="", key="", location="", repo_id=""):
|
||||
self.backupper.set_creds(login, key, location)
|
||||
self.login = login
|
||||
self.key = key
|
||||
self.location = location
|
||||
# We do not need to do anything with this one
|
||||
# Just remember in case the app forgets
|
||||
self.repo_id = repo_id
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
"""
|
||||
Module for storing backup related data in redis.
|
||||
"""
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.models.backup.provider import BackupProviderModel
|
||||
from selfprivacy_api.graphql.common_types.backup import (
|
||||
AutobackupQuotas,
|
||||
_AutobackupQuotas,
|
||||
)
|
||||
|
||||
from selfprivacy_api.utils.redis_pool import RedisPool
|
||||
from selfprivacy_api.utils.redis_model_storage import (
|
||||
store_model_as_hash,
|
||||
hash_as_model,
|
||||
)
|
||||
|
||||
from selfprivacy_api.backup.providers.provider import AbstractBackupProvider
|
||||
from selfprivacy_api.backup.providers import get_kind
|
||||
|
||||
REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:"
|
||||
REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:"
|
||||
REDIS_INITTED_CACHE = "backups:repo_initted"
|
||||
|
||||
REDIS_PROVIDER_KEY = "backups:provider"
|
||||
REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period"
|
||||
|
||||
REDIS_AUTOBACKUP_QUOTAS_KEY = "backups:autobackup_quotas_key"
|
||||
|
||||
redis = RedisPool().get_connection()
|
||||
|
||||
|
||||
class Storage:
|
||||
"""Static class for storing backup related data in redis"""
|
||||
|
||||
@staticmethod
|
||||
def reset() -> None:
|
||||
"""Deletes all backup related data from redis"""
|
||||
redis.delete(REDIS_PROVIDER_KEY)
|
||||
redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
|
||||
redis.delete(REDIS_INITTED_CACHE)
|
||||
redis.delete(REDIS_AUTOBACKUP_QUOTAS_KEY)
|
||||
|
||||
prefixes_to_clean = [
|
||||
REDIS_SNAPSHOTS_PREFIX,
|
||||
REDIS_LAST_BACKUP_PREFIX,
|
||||
]
|
||||
|
||||
for prefix in prefixes_to_clean:
|
||||
for key in redis.keys(prefix + "*"):
|
||||
redis.delete(key)
|
||||
|
||||
@staticmethod
|
||||
def invalidate_snapshot_storage() -> None:
|
||||
"""Deletes all cached snapshots from redis"""
|
||||
for key in redis.keys(REDIS_SNAPSHOTS_PREFIX + "*"):
|
||||
redis.delete(key)
|
||||
|
||||
@staticmethod
|
||||
def __last_backup_key(service_id: str) -> str:
|
||||
return REDIS_LAST_BACKUP_PREFIX + service_id
|
||||
|
||||
@staticmethod
|
||||
def __snapshot_key(snapshot: Snapshot) -> str:
|
||||
return REDIS_SNAPSHOTS_PREFIX + snapshot.id
|
||||
|
||||
@staticmethod
|
||||
def get_last_backup_time(service_id: str) -> Optional[datetime]:
|
||||
"""Returns last backup time for a service or None if it was never backed up"""
|
||||
key = Storage.__last_backup_key(service_id)
|
||||
if not redis.exists(key):
|
||||
return None
|
||||
|
||||
snapshot = hash_as_model(redis, key, Snapshot)
|
||||
if not snapshot:
|
||||
return None
|
||||
return snapshot.created_at
|
||||
|
||||
@staticmethod
|
||||
def store_last_timestamp(service_id: str, snapshot: Snapshot) -> None:
|
||||
"""Stores last backup time for a service"""
|
||||
store_model_as_hash(
|
||||
redis,
|
||||
Storage.__last_backup_key(service_id),
|
||||
snapshot,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def cache_snapshot(snapshot: Snapshot) -> None:
|
||||
"""Stores snapshot metadata in redis for caching purposes"""
|
||||
snapshot_key = Storage.__snapshot_key(snapshot)
|
||||
store_model_as_hash(redis, snapshot_key, snapshot)
|
||||
|
||||
@staticmethod
|
||||
def delete_cached_snapshot(snapshot: Snapshot) -> None:
|
||||
"""Deletes snapshot metadata from redis"""
|
||||
snapshot_key = Storage.__snapshot_key(snapshot)
|
||||
redis.delete(snapshot_key)
|
||||
|
||||
@staticmethod
|
||||
def get_cached_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]:
|
||||
"""Returns cached snapshot by id or None if it doesn't exist"""
|
||||
key = REDIS_SNAPSHOTS_PREFIX + snapshot_id
|
||||
if not redis.exists(key):
|
||||
return None
|
||||
return hash_as_model(redis, key, Snapshot)
|
||||
|
||||
@staticmethod
|
||||
def get_cached_snapshots() -> List[Snapshot]:
|
||||
"""Returns all cached snapshots stored in redis"""
|
||||
keys: list[str] = redis.keys(REDIS_SNAPSHOTS_PREFIX + "*") # type: ignore
|
||||
result: list[Snapshot] = []
|
||||
|
||||
for key in keys:
|
||||
snapshot = hash_as_model(redis, key, Snapshot)
|
||||
if snapshot:
|
||||
result.append(snapshot)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def autobackup_period_minutes() -> Optional[int]:
|
||||
"""None means autobackup is disabled"""
|
||||
if not redis.exists(REDIS_AUTOBACKUP_PERIOD_KEY):
|
||||
return None
|
||||
return int(redis.get(REDIS_AUTOBACKUP_PERIOD_KEY)) # type: ignore
|
||||
|
||||
@staticmethod
|
||||
def store_autobackup_period_minutes(minutes: int) -> None:
|
||||
"""Set the new autobackup period in minutes"""
|
||||
redis.set(REDIS_AUTOBACKUP_PERIOD_KEY, minutes)
|
||||
|
||||
@staticmethod
|
||||
def delete_backup_period() -> None:
|
||||
"""Set the autobackup period to none, effectively disabling autobackup"""
|
||||
redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
|
||||
|
||||
@staticmethod
|
||||
def store_provider(provider: AbstractBackupProvider) -> None:
|
||||
"""Stores backup provider auth data in redis"""
|
||||
model = BackupProviderModel(
|
||||
kind=get_kind(provider),
|
||||
login=provider.login,
|
||||
key=provider.key,
|
||||
location=provider.location,
|
||||
repo_id=provider.repo_id,
|
||||
)
|
||||
store_model_as_hash(redis, REDIS_PROVIDER_KEY, model)
|
||||
if Storage.load_provider() != model:
|
||||
raise IOError("could not store the provider model: ", model.dict)
|
||||
|
||||
@staticmethod
|
||||
def load_provider() -> Optional[BackupProviderModel]:
|
||||
"""Loads backup storage provider auth data from redis"""
|
||||
provider_model = hash_as_model(
|
||||
redis,
|
||||
REDIS_PROVIDER_KEY,
|
||||
BackupProviderModel,
|
||||
)
|
||||
return provider_model
|
||||
|
||||
@staticmethod
|
||||
def has_init_mark() -> bool:
|
||||
"""Returns True if the repository was initialized"""
|
||||
if redis.exists(REDIS_INITTED_CACHE):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def mark_as_init():
|
||||
"""Marks the repository as initialized"""
|
||||
redis.set(REDIS_INITTED_CACHE, 1)
|
||||
|
||||
@staticmethod
|
||||
def mark_as_uninitted():
|
||||
"""Marks the repository as initialized"""
|
||||
redis.delete(REDIS_INITTED_CACHE)
|
||||
|
||||
@staticmethod
|
||||
def set_autobackup_quotas(quotas: AutobackupQuotas) -> None:
|
||||
store_model_as_hash(redis, REDIS_AUTOBACKUP_QUOTAS_KEY, quotas.to_pydantic())
|
||||
|
||||
@staticmethod
|
||||
def autobackup_quotas() -> AutobackupQuotas:
|
||||
quotas_model = hash_as_model(
|
||||
redis, REDIS_AUTOBACKUP_QUOTAS_KEY, _AutobackupQuotas
|
||||
)
|
||||
if quotas_model is None:
|
||||
unlimited_quotas = AutobackupQuotas(
|
||||
last=-1,
|
||||
daily=-1,
|
||||
weekly=-1,
|
||||
monthly=-1,
|
||||
yearly=-1,
|
||||
)
|
||||
return unlimited_quotas
|
||||
return AutobackupQuotas.from_pydantic(quotas_model) # pylint: disable=no-member
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
"""
|
||||
The tasks module contains the worker tasks that are used to back up and restore
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from selfprivacy_api.graphql.common_types.backup import (
|
||||
RestoreStrategy,
|
||||
BackupReason,
|
||||
)
|
||||
|
||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
from huey import crontab
|
||||
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
from selfprivacy_api.backup import Backups
|
||||
from selfprivacy_api.backup.jobs import add_autobackup_job
|
||||
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
||||
|
||||
|
||||
SNAPSHOT_CACHE_TTL_HOURS = 6
|
||||
|
||||
|
||||
def validate_datetime(dt: datetime) -> bool:
|
||||
"""
|
||||
Validates that it is time to back up.
|
||||
Also ensures that the timezone-aware time is used.
|
||||
"""
|
||||
if dt.tzinfo is None:
|
||||
return Backups.is_time_to_backup(dt.replace(tzinfo=timezone.utc))
|
||||
return Backups.is_time_to_backup(dt)
|
||||
|
||||
|
||||
# huey tasks need to return something
|
||||
@huey.task()
|
||||
def start_backup(service_id: str, reason: BackupReason = BackupReason.EXPLICIT) -> bool:
|
||||
"""
|
||||
The worker task that starts the backup process.
|
||||
"""
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
raise ValueError(f"No such service: {service_id}")
|
||||
Backups.back_up(service, reason)
|
||||
return True
|
||||
|
||||
|
||||
@huey.task()
|
||||
def prune_autobackup_snapshots(job: Job) -> bool:
|
||||
"""
|
||||
Remove all autobackup snapshots that do not fit into quotas set
|
||||
"""
|
||||
Jobs.update(job, JobStatus.RUNNING)
|
||||
try:
|
||||
Backups.prune_all_autosnaps()
|
||||
except Exception as e:
|
||||
Jobs.update(job, JobStatus.ERROR, error=type(e).__name__ + ":" + str(e))
|
||||
return False
|
||||
|
||||
Jobs.update(job, JobStatus.FINISHED)
|
||||
return True
|
||||
|
||||
|
||||
@huey.task()
|
||||
def restore_snapshot(
|
||||
snapshot: Snapshot,
|
||||
strategy: RestoreStrategy = RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE,
|
||||
) -> bool:
|
||||
"""
|
||||
The worker task that starts the restore process.
|
||||
"""
|
||||
Backups.restore_snapshot(snapshot, strategy)
|
||||
return True
|
||||
|
||||
|
||||
def do_autobackup() -> None:
|
||||
"""
|
||||
Body of autobackup task, broken out to test it
|
||||
For some reason, we cannot launch periodic huey tasks
|
||||
inside tests
|
||||
"""
|
||||
time = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||
services_to_back_up = Backups.services_to_back_up(time)
|
||||
if not services_to_back_up:
|
||||
return
|
||||
job = add_autobackup_job(services_to_back_up)
|
||||
|
||||
progress_per_service = 100 // len(services_to_back_up)
|
||||
progress = 0
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
|
||||
for service in services_to_back_up:
|
||||
try:
|
||||
Backups.back_up(service, BackupReason.AUTO)
|
||||
except Exception as error:
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(error).__name__ + ": " + str(error),
|
||||
)
|
||||
return
|
||||
progress = progress + progress_per_service
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
|
||||
Jobs.update(job, JobStatus.FINISHED)
|
||||
|
||||
|
||||
@huey.periodic_task(validate_datetime=validate_datetime)
|
||||
def automatic_backup() -> None:
|
||||
"""
|
||||
The worker periodic task that starts the automatic backup process.
|
||||
"""
|
||||
do_autobackup()
|
||||
|
||||
|
||||
@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
|
||||
def reload_snapshot_cache():
|
||||
Backups.force_snapshot_cache_reload()
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
import subprocess
|
||||
from os.path import exists
|
||||
from typing import Generator
|
||||
|
||||
|
||||
def output_yielder(command) -> Generator[str, None, None]:
|
||||
"""Note: If you break during iteration, it kills the process"""
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
) as handle:
|
||||
if handle is None or handle.stdout is None:
|
||||
raise ValueError("could not run command: ", command)
|
||||
|
||||
try:
|
||||
for line in iter(handle.stdout.readline, ""):
|
||||
if "NOTICE:" not in line:
|
||||
yield line
|
||||
except GeneratorExit:
|
||||
handle.kill()
|
||||
|
||||
|
||||
def sync(src_path: str, dest_path: str):
|
||||
"""a wrapper around rclone sync"""
|
||||
|
||||
if not exists(src_path):
|
||||
raise ValueError("source dir for rclone sync must exist")
|
||||
|
||||
rclone_command = ["rclone", "sync", "-P", src_path, dest_path]
|
||||
for raw_message in output_yielder(rclone_command):
|
||||
if "ERROR" in raw_message:
|
||||
raise ValueError(raw_message)
|
||||
|
|
@ -2,7 +2,7 @@ from fastapi import Depends, HTTPException, status
|
|||
from fastapi.security import APIKeyHeader
|
||||
from pydantic import BaseModel
|
||||
|
||||
from selfprivacy_api.actions.api_tokens import is_token_valid
|
||||
from selfprivacy_api.utils.auth import is_token_valid
|
||||
|
||||
|
||||
class TokenHeader(BaseModel):
|
||||
|
|
@ -27,4 +27,4 @@ async def get_token_header(
|
|||
|
||||
def get_api_version() -> str:
|
||||
"""Get API version"""
|
||||
return "3.1.0"
|
||||
return "2.0.9"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import typing
|
|||
from strawberry.permission import BasePermission
|
||||
from strawberry.types import Info
|
||||
|
||||
from selfprivacy_api.actions.api_tokens import is_token_valid
|
||||
from selfprivacy_api.utils.auth import is_token_valid
|
||||
|
||||
|
||||
class IsAuthenticated(BasePermission):
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
"""Backup"""
|
||||
# pylint: disable=too-few-public-methods
|
||||
from enum import Enum
|
||||
import strawberry
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
@strawberry.enum
|
||||
class RestoreStrategy(Enum):
|
||||
INPLACE = "INPLACE"
|
||||
DOWNLOAD_VERIFY_OVERWRITE = "DOWNLOAD_VERIFY_OVERWRITE"
|
||||
|
||||
|
||||
@strawberry.enum
|
||||
class BackupReason(Enum):
|
||||
EXPLICIT = "EXPLICIT"
|
||||
AUTO = "AUTO"
|
||||
PRE_RESTORE = "PRE_RESTORE"
|
||||
|
||||
|
||||
class _AutobackupQuotas(BaseModel):
|
||||
last: int
|
||||
daily: int
|
||||
weekly: int
|
||||
monthly: int
|
||||
yearly: int
|
||||
|
||||
|
||||
@strawberry.experimental.pydantic.type(model=_AutobackupQuotas, all_fields=True)
|
||||
class AutobackupQuotas:
|
||||
pass
|
||||
|
||||
|
||||
@strawberry.experimental.pydantic.input(model=_AutobackupQuotas, all_fields=True)
|
||||
class AutobackupQuotasInput:
|
||||
pass
|
||||
|
|
@ -2,7 +2,6 @@ import typing
|
|||
import strawberry
|
||||
|
||||
|
||||
# TODO: use https://strawberry.rocks/docs/integrations/pydantic when it is stable
|
||||
@strawberry.type
|
||||
class DnsRecord:
|
||||
"""DNS record"""
|
||||
|
|
@ -12,4 +11,3 @@ class DnsRecord:
|
|||
content: str
|
||||
ttl: int
|
||||
priority: typing.Optional[int]
|
||||
display_name: str
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ class ApiJob:
|
|||
"""Job type for GraphQL."""
|
||||
|
||||
uid: str
|
||||
type_id: str
|
||||
name: str
|
||||
description: str
|
||||
status: str
|
||||
|
|
@ -29,7 +28,6 @@ def job_to_api_job(job: Job) -> ApiJob:
|
|||
"""Convert a Job from jobs controller to a GraphQL ApiJob."""
|
||||
return ApiJob(
|
||||
uid=str(job.uid),
|
||||
type_id=job.type_id,
|
||||
name=job.name,
|
||||
description=job.description,
|
||||
status=job.status.name,
|
||||
|
|
@ -45,7 +43,7 @@ def job_to_api_job(job: Job) -> ApiJob:
|
|||
|
||||
def get_api_job_by_id(job_id: str) -> typing.Optional[ApiJob]:
|
||||
"""Get a job for GraphQL by its ID."""
|
||||
job = Jobs.get_job(job_id)
|
||||
job = Jobs.get_instance().get_job(job_id)
|
||||
if job is None:
|
||||
return None
|
||||
return job_to_api_job(job)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,11 @@
|
|||
from enum import Enum
|
||||
from typing import Optional, List
|
||||
import datetime
|
||||
import typing
|
||||
import strawberry
|
||||
|
||||
from selfprivacy_api.graphql.common_types.backup import BackupReason
|
||||
from selfprivacy_api.graphql.common_types.dns import DnsRecord
|
||||
|
||||
from selfprivacy_api.services import get_service_by_id, get_services_by_location
|
||||
from selfprivacy_api.services import Service as ServiceInterface
|
||||
from selfprivacy_api.services import ServiceDnsRecord
|
||||
|
||||
from selfprivacy_api.utils.block_devices import BlockDevices
|
||||
from selfprivacy_api.utils.network import get_ip4, get_ip6
|
||||
|
||||
|
||||
def get_usages(root: "StorageVolume") -> list["StorageUsageInterface"]:
|
||||
|
|
@ -21,7 +15,7 @@ def get_usages(root: "StorageVolume") -> list["StorageUsageInterface"]:
|
|||
service=service_to_graphql_service(service),
|
||||
title=service.get_display_name(),
|
||||
used_space=str(service.get_storage_usage()),
|
||||
volume=get_volume_by_id(service.get_drive()),
|
||||
volume=get_volume_by_id(service.get_location()),
|
||||
)
|
||||
for service in get_services_by_location(root.name)
|
||||
]
|
||||
|
|
@ -36,8 +30,8 @@ class StorageVolume:
|
|||
used_space: str
|
||||
root: bool
|
||||
name: str
|
||||
model: Optional[str]
|
||||
serial: Optional[str]
|
||||
model: typing.Optional[str]
|
||||
serial: typing.Optional[str]
|
||||
type: str
|
||||
|
||||
@strawberry.field
|
||||
|
|
@ -49,7 +43,7 @@ class StorageVolume:
|
|||
@strawberry.interface
|
||||
class StorageUsageInterface:
|
||||
used_space: str
|
||||
volume: Optional[StorageVolume]
|
||||
volume: typing.Optional[StorageVolume]
|
||||
title: str
|
||||
|
||||
|
||||
|
|
@ -57,7 +51,7 @@ class StorageUsageInterface:
|
|||
class ServiceStorageUsage(StorageUsageInterface):
|
||||
"""Storage usage for a service"""
|
||||
|
||||
service: Optional["Service"]
|
||||
service: typing.Optional["Service"]
|
||||
|
||||
|
||||
@strawberry.enum
|
||||
|
|
@ -85,21 +79,7 @@ def get_storage_usage(root: "Service") -> ServiceStorageUsage:
|
|||
service=service_to_graphql_service(service),
|
||||
title=service.get_display_name(),
|
||||
used_space=str(service.get_storage_usage()),
|
||||
volume=get_volume_by_id(service.get_drive()),
|
||||
)
|
||||
|
||||
|
||||
# TODO: This won't be needed when deriving DnsRecord via strawberry pydantic integration
|
||||
# https://strawberry.rocks/docs/integrations/pydantic
|
||||
# Remove when the link above says it got stable.
|
||||
def service_dns_to_graphql(record: ServiceDnsRecord) -> DnsRecord:
|
||||
return DnsRecord(
|
||||
record_type=record.type,
|
||||
name=record.name,
|
||||
content=record.content,
|
||||
ttl=record.ttl,
|
||||
priority=record.priority,
|
||||
display_name=record.display_name,
|
||||
volume=get_volume_by_id(service.get_location()),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -112,39 +92,15 @@ class Service:
|
|||
is_movable: bool
|
||||
is_required: bool
|
||||
is_enabled: bool
|
||||
can_be_backed_up: bool
|
||||
backup_description: str
|
||||
status: ServiceStatusEnum
|
||||
url: Optional[str]
|
||||
|
||||
@strawberry.field
|
||||
def dns_records(self) -> Optional[List[DnsRecord]]:
|
||||
service = get_service_by_id(self.id)
|
||||
if service is None:
|
||||
raise LookupError(f"no service {self.id}. Should be unreachable")
|
||||
|
||||
raw_records = service.get_dns_records(get_ip4(), get_ip6())
|
||||
dns_records = [service_dns_to_graphql(record) for record in raw_records]
|
||||
return dns_records
|
||||
url: typing.Optional[str]
|
||||
dns_records: typing.Optional[typing.List[DnsRecord]]
|
||||
|
||||
@strawberry.field
|
||||
def storage_usage(self) -> ServiceStorageUsage:
|
||||
"""Get storage usage for a service"""
|
||||
return get_storage_usage(self)
|
||||
|
||||
# TODO: fill this
|
||||
@strawberry.field
|
||||
def backup_snapshots(self) -> Optional[List["SnapshotInfo"]]:
|
||||
return None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SnapshotInfo:
|
||||
id: str
|
||||
service: Service
|
||||
created_at: datetime.datetime
|
||||
reason: BackupReason
|
||||
|
||||
|
||||
def service_to_graphql_service(service: ServiceInterface) -> Service:
|
||||
"""Convert service to graphql service"""
|
||||
|
|
@ -156,14 +112,22 @@ def service_to_graphql_service(service: ServiceInterface) -> Service:
|
|||
is_movable=service.is_movable(),
|
||||
is_required=service.is_required(),
|
||||
is_enabled=service.is_enabled(),
|
||||
can_be_backed_up=service.can_be_backed_up(),
|
||||
backup_description=service.get_backup_description(),
|
||||
status=ServiceStatusEnum(service.get_status().value),
|
||||
url=service.get_url(),
|
||||
dns_records=[
|
||||
DnsRecord(
|
||||
record_type=record.type,
|
||||
name=record.name,
|
||||
content=record.content,
|
||||
ttl=record.ttl,
|
||||
priority=record.priority,
|
||||
)
|
||||
for record in service.get_dns_records()
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def get_volume_by_id(volume_id: str) -> Optional[StorageVolume]:
|
||||
def get_volume_by_id(volume_id: str) -> typing.Optional[StorageVolume]:
|
||||
"""Get volume by id"""
|
||||
volume = BlockDevices().get_block_device(volume_id)
|
||||
if volume is None:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class UserType(Enum):
|
|||
|
||||
@strawberry.type
|
||||
class User:
|
||||
|
||||
user_type: UserType
|
||||
username: str
|
||||
# userHomeFolderspace: UserHomeFolderUsage
|
||||
|
|
@ -31,6 +32,7 @@ class UserMutationReturn(MutationReturnInterface):
|
|||
|
||||
|
||||
def get_user_by_username(username: str) -> typing.Optional[User]:
|
||||
|
||||
user = users_actions.get_user_by_username(username)
|
||||
if user is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -11,11 +11,6 @@ from selfprivacy_api.actions.api_tokens import (
|
|||
NotFoundException,
|
||||
delete_api_token,
|
||||
get_new_api_recovery_key,
|
||||
use_mnemonic_recovery_token,
|
||||
refresh_api_token,
|
||||
delete_new_device_auth_token,
|
||||
get_new_device_auth_token,
|
||||
use_new_device_auth_token,
|
||||
)
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
|
|
@ -23,6 +18,14 @@ from selfprivacy_api.graphql.mutations.mutation_interface import (
|
|||
MutationReturnInterface,
|
||||
)
|
||||
|
||||
from selfprivacy_api.utils.auth import (
|
||||
delete_new_device_auth_token,
|
||||
get_new_device_auth_token,
|
||||
refresh_token,
|
||||
use_mnemonic_recoverery_token,
|
||||
use_new_device_auth_token,
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class ApiKeyMutationReturn(MutationReturnInterface):
|
||||
|
|
@ -95,53 +98,50 @@ class ApiMutations:
|
|||
self, input: UseRecoveryKeyInput
|
||||
) -> DeviceApiTokenMutationReturn:
|
||||
"""Use recovery key"""
|
||||
token = use_mnemonic_recovery_token(input.key, input.deviceName)
|
||||
if token is not None:
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=True,
|
||||
message="Recovery key used",
|
||||
code=200,
|
||||
token=token,
|
||||
)
|
||||
else:
|
||||
token = use_mnemonic_recoverery_token(input.key, input.deviceName)
|
||||
if token is None:
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=False,
|
||||
message="Recovery key not found",
|
||||
code=404,
|
||||
token=None,
|
||||
)
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=True,
|
||||
message="Recovery key used",
|
||||
code=200,
|
||||
token=token,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def refresh_device_api_token(self, info: Info) -> DeviceApiTokenMutationReturn:
|
||||
"""Refresh device api token"""
|
||||
token_string = (
|
||||
token = (
|
||||
info.context["request"]
|
||||
.headers.get("Authorization", "")
|
||||
.replace("Bearer ", "")
|
||||
)
|
||||
if token_string is None:
|
||||
if token is None:
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=False,
|
||||
message="Token not found",
|
||||
code=404,
|
||||
token=None,
|
||||
)
|
||||
|
||||
try:
|
||||
new_token = refresh_api_token(token_string)
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=True,
|
||||
message="Token refreshed",
|
||||
code=200,
|
||||
token=new_token,
|
||||
)
|
||||
except NotFoundException:
|
||||
new_token = refresh_token(token)
|
||||
if new_token is None:
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=False,
|
||||
message="Token not found",
|
||||
code=404,
|
||||
token=None,
|
||||
)
|
||||
return DeviceApiTokenMutationReturn(
|
||||
success=True,
|
||||
message="Token refreshed",
|
||||
code=200,
|
||||
token=new_token,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def delete_device_api_token(self, device: str, info: Info) -> GenericMutationReturn:
|
||||
|
|
|
|||
|
|
@ -1,241 +0,0 @@
|
|||
import typing
|
||||
import strawberry
|
||||
|
||||
from selfprivacy_api.jobs import Jobs
|
||||
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericMutationReturn,
|
||||
GenericJobMutationReturn,
|
||||
MutationReturnInterface,
|
||||
)
|
||||
from selfprivacy_api.graphql.queries.backup import BackupConfiguration
|
||||
from selfprivacy_api.graphql.queries.backup import Backup
|
||||
from selfprivacy_api.graphql.queries.providers import BackupProvider
|
||||
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||
from selfprivacy_api.graphql.common_types.backup import (
|
||||
AutobackupQuotasInput,
|
||||
RestoreStrategy,
|
||||
)
|
||||
|
||||
from selfprivacy_api.backup import Backups
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
from selfprivacy_api.backup.tasks import (
|
||||
start_backup,
|
||||
restore_snapshot,
|
||||
prune_autobackup_snapshots,
|
||||
)
|
||||
from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job
|
||||
|
||||
|
||||
@strawberry.input
|
||||
class InitializeRepositoryInput:
|
||||
"""Initialize repository input"""
|
||||
|
||||
provider: BackupProvider
|
||||
# The following field may become optional for other providers?
|
||||
# Backblaze takes bucket id and name
|
||||
location_id: str
|
||||
location_name: str
|
||||
# Key ID and key for Backblaze
|
||||
login: str
|
||||
password: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class GenericBackupConfigReturn(MutationReturnInterface):
|
||||
"""Generic backup config return"""
|
||||
|
||||
configuration: typing.Optional[BackupConfiguration]
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class BackupMutations:
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def initialize_repository(
|
||||
self, repository: InitializeRepositoryInput
|
||||
) -> GenericBackupConfigReturn:
|
||||
"""Initialize a new repository"""
|
||||
Backups.set_provider(
|
||||
kind=repository.provider,
|
||||
login=repository.login,
|
||||
key=repository.password,
|
||||
location=repository.location_name,
|
||||
repo_id=repository.location_id,
|
||||
)
|
||||
Backups.init_repo()
|
||||
return GenericBackupConfigReturn(
|
||||
success=True,
|
||||
message="",
|
||||
code=200,
|
||||
configuration=Backup().configuration(),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def remove_repository(self) -> GenericBackupConfigReturn:
|
||||
"""Remove repository"""
|
||||
Backups.reset()
|
||||
return GenericBackupConfigReturn(
|
||||
success=True,
|
||||
message="",
|
||||
code=200,
|
||||
configuration=Backup().configuration(),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def set_autobackup_period(
|
||||
self, period: typing.Optional[int] = None
|
||||
) -> GenericBackupConfigReturn:
|
||||
"""Set autobackup period. None is to disable autobackup"""
|
||||
if period is not None:
|
||||
Backups.set_autobackup_period_minutes(period)
|
||||
else:
|
||||
Backups.set_autobackup_period_minutes(0)
|
||||
|
||||
return GenericBackupConfigReturn(
|
||||
success=True,
|
||||
message="",
|
||||
code=200,
|
||||
configuration=Backup().configuration(),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def set_autobackup_quotas(
|
||||
self, quotas: AutobackupQuotasInput
|
||||
) -> GenericBackupConfigReturn:
|
||||
"""
|
||||
Set autobackup quotas.
|
||||
Values <=0 for any timeframe mean no limits for that timeframe.
|
||||
To disable autobackup use autobackup period setting, not this mutation.
|
||||
"""
|
||||
|
||||
job = Jobs.add(
|
||||
name="Trimming autobackup snapshots",
|
||||
type_id="backups.autobackup_trimming",
|
||||
description="Pruning the excessive snapshots after the new autobackup quotas are set",
|
||||
)
|
||||
|
||||
try:
|
||||
Backups.set_autobackup_quotas(quotas)
|
||||
# this task is async and can fail with only a job to report the error
|
||||
prune_autobackup_snapshots(job)
|
||||
return GenericBackupConfigReturn(
|
||||
success=True,
|
||||
message="",
|
||||
code=200,
|
||||
configuration=Backup().configuration(),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return GenericBackupConfigReturn(
|
||||
success=False,
|
||||
message=type(e).__name__ + ":" + str(e),
|
||||
code=400,
|
||||
configuration=Backup().configuration(),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def start_backup(self, service_id: str) -> GenericJobMutationReturn:
|
||||
"""Start backup"""
|
||||
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
code=300,
|
||||
message=f"nonexistent service: {service_id}",
|
||||
job=None,
|
||||
)
|
||||
|
||||
job = add_backup_job(service)
|
||||
start_backup(service_id)
|
||||
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="Backup job queued",
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def restore_backup(
|
||||
self,
|
||||
snapshot_id: str,
|
||||
strategy: RestoreStrategy = RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE,
|
||||
) -> GenericJobMutationReturn:
|
||||
"""Restore backup"""
|
||||
snap = Backups.get_snapshot_by_id(snapshot_id)
|
||||
if snap is None:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
code=404,
|
||||
message=f"No such snapshot: {snapshot_id}",
|
||||
job=None,
|
||||
)
|
||||
|
||||
service = get_service_by_id(snap.service_name)
|
||||
if service is None:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
code=404,
|
||||
message=f"nonexistent service: {snap.service_name}",
|
||||
job=None,
|
||||
)
|
||||
|
||||
try:
|
||||
job = add_restore_job(snap)
|
||||
except ValueError as error:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
code=400,
|
||||
message=str(error),
|
||||
job=None,
|
||||
)
|
||||
|
||||
restore_snapshot(snap, strategy)
|
||||
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="restore job created",
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def forget_snapshot(self, snapshot_id: str) -> GenericMutationReturn:
|
||||
"""Forget a snapshot.
|
||||
Makes it inaccessible from the server.
|
||||
After some time, the data (encrypted) will not be recoverable
|
||||
from the backup server too, but not immediately"""
|
||||
|
||||
snap = Backups.get_snapshot_by_id(snapshot_id)
|
||||
if snap is None:
|
||||
return GenericMutationReturn(
|
||||
success=False,
|
||||
code=404,
|
||||
message=f"snapshot {snapshot_id} not found",
|
||||
)
|
||||
|
||||
try:
|
||||
Backups.forget_snapshot(snap)
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="",
|
||||
)
|
||||
except Exception as error:
|
||||
return GenericMutationReturn(
|
||||
success=False,
|
||||
code=400,
|
||||
message=str(error),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def force_snapshots_reload(self) -> GenericMutationReturn:
|
||||
"""Force snapshots reload"""
|
||||
Backups.force_snapshot_cache_reload()
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="",
|
||||
)
|
||||
|
|
@ -1,216 +0,0 @@
|
|||
"""Deprecated mutations
|
||||
|
||||
There was made a mistake, where mutations were not grouped, and were instead
|
||||
placed in the root of mutations schema. In this file, we import all the
|
||||
mutations from and provide them to the root for backwards compatibility.
|
||||
"""
|
||||
|
||||
import strawberry
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.common_types.user import UserMutationReturn
|
||||
from selfprivacy_api.graphql.mutations.api_mutations import (
|
||||
ApiKeyMutationReturn,
|
||||
ApiMutations,
|
||||
DeviceApiTokenMutationReturn,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
|
||||
from selfprivacy_api.graphql.mutations.job_mutations import JobMutations
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobMutationReturn,
|
||||
GenericMutationReturn,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.services_mutations import (
|
||||
ServiceJobMutationReturn,
|
||||
ServiceMutationReturn,
|
||||
ServicesMutations,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.storage_mutations import StorageMutations
|
||||
from selfprivacy_api.graphql.mutations.system_mutations import (
|
||||
AutoUpgradeSettingsMutationReturn,
|
||||
SystemMutations,
|
||||
TimezoneMutationReturn,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
|
||||
from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
|
||||
|
||||
|
||||
def deprecated_mutation(func, group, auth=True):
|
||||
return strawberry.mutation(
|
||||
resolver=func,
|
||||
permission_classes=[IsAuthenticated] if auth else [],
|
||||
deprecation_reason=f"Use `{group}.{func.__name__}` instead",
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedApiMutations:
|
||||
get_new_recovery_api_key: ApiKeyMutationReturn = deprecated_mutation(
|
||||
ApiMutations.get_new_recovery_api_key,
|
||||
"api",
|
||||
)
|
||||
|
||||
use_recovery_api_key: DeviceApiTokenMutationReturn = deprecated_mutation(
|
||||
ApiMutations.use_recovery_api_key,
|
||||
"api",
|
||||
auth=False,
|
||||
)
|
||||
|
||||
refresh_device_api_token: DeviceApiTokenMutationReturn = deprecated_mutation(
|
||||
ApiMutations.refresh_device_api_token,
|
||||
"api",
|
||||
)
|
||||
|
||||
delete_device_api_token: GenericMutationReturn = deprecated_mutation(
|
||||
ApiMutations.delete_device_api_token,
|
||||
"api",
|
||||
)
|
||||
|
||||
get_new_device_api_key: ApiKeyMutationReturn = deprecated_mutation(
|
||||
ApiMutations.get_new_device_api_key,
|
||||
"api",
|
||||
)
|
||||
|
||||
invalidate_new_device_api_key: GenericMutationReturn = deprecated_mutation(
|
||||
ApiMutations.invalidate_new_device_api_key,
|
||||
"api",
|
||||
)
|
||||
|
||||
authorize_with_new_device_api_key: DeviceApiTokenMutationReturn = (
|
||||
deprecated_mutation(
|
||||
ApiMutations.authorize_with_new_device_api_key,
|
||||
"api",
|
||||
auth=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedSystemMutations:
|
||||
change_timezone: TimezoneMutationReturn = deprecated_mutation(
|
||||
SystemMutations.change_timezone,
|
||||
"system",
|
||||
)
|
||||
|
||||
change_auto_upgrade_settings: AutoUpgradeSettingsMutationReturn = (
|
||||
deprecated_mutation(
|
||||
SystemMutations.change_auto_upgrade_settings,
|
||||
"system",
|
||||
)
|
||||
)
|
||||
|
||||
run_system_rebuild: GenericMutationReturn = deprecated_mutation(
|
||||
SystemMutations.run_system_rebuild,
|
||||
"system",
|
||||
)
|
||||
|
||||
run_system_rollback: GenericMutationReturn = deprecated_mutation(
|
||||
SystemMutations.run_system_rollback,
|
||||
"system",
|
||||
)
|
||||
|
||||
run_system_upgrade: GenericMutationReturn = deprecated_mutation(
|
||||
SystemMutations.run_system_upgrade,
|
||||
"system",
|
||||
)
|
||||
|
||||
reboot_system: GenericMutationReturn = deprecated_mutation(
|
||||
SystemMutations.reboot_system,
|
||||
"system",
|
||||
)
|
||||
|
||||
pull_repository_changes: GenericMutationReturn = deprecated_mutation(
|
||||
SystemMutations.pull_repository_changes,
|
||||
"system",
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedUsersMutations:
|
||||
create_user: UserMutationReturn = deprecated_mutation(
|
||||
UsersMutations.create_user,
|
||||
"users",
|
||||
)
|
||||
|
||||
delete_user: GenericMutationReturn = deprecated_mutation(
|
||||
UsersMutations.delete_user,
|
||||
"users",
|
||||
)
|
||||
|
||||
update_user: UserMutationReturn = deprecated_mutation(
|
||||
UsersMutations.update_user,
|
||||
"users",
|
||||
)
|
||||
|
||||
add_ssh_key: UserMutationReturn = deprecated_mutation(
|
||||
UsersMutations.add_ssh_key,
|
||||
"users",
|
||||
)
|
||||
|
||||
remove_ssh_key: UserMutationReturn = deprecated_mutation(
|
||||
UsersMutations.remove_ssh_key,
|
||||
"users",
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedStorageMutations:
|
||||
resize_volume: GenericMutationReturn = deprecated_mutation(
|
||||
StorageMutations.resize_volume,
|
||||
"storage",
|
||||
)
|
||||
|
||||
mount_volume: GenericMutationReturn = deprecated_mutation(
|
||||
StorageMutations.mount_volume,
|
||||
"storage",
|
||||
)
|
||||
|
||||
unmount_volume: GenericMutationReturn = deprecated_mutation(
|
||||
StorageMutations.unmount_volume,
|
||||
"storage",
|
||||
)
|
||||
|
||||
migrate_to_binds: GenericJobMutationReturn = deprecated_mutation(
|
||||
StorageMutations.migrate_to_binds,
|
||||
"storage",
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedServicesMutations:
|
||||
enable_service: ServiceMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.enable_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
disable_service: ServiceMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.disable_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
stop_service: ServiceMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.stop_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
start_service: ServiceMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.start_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
restart_service: ServiceMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.restart_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
move_service: ServiceJobMutationReturn = deprecated_mutation(
|
||||
ServicesMutations.move_service,
|
||||
"services",
|
||||
)
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class DeprecatedJobMutations:
|
||||
remove_job: GenericMutationReturn = deprecated_mutation(
|
||||
JobMutations.remove_job,
|
||||
"jobs",
|
||||
)
|
||||
|
|
@ -14,7 +14,7 @@ class JobMutations:
|
|||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def remove_job(self, job_id: str) -> GenericMutationReturn:
|
||||
"""Remove a job from the queue"""
|
||||
result = Jobs.remove_by_uid(job_id)
|
||||
result = Jobs.get_instance().remove_by_uid(job_id)
|
||||
if result:
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
|
|
|
|||
|
|
@ -17,5 +17,5 @@ class GenericMutationReturn(MutationReturnInterface):
|
|||
|
||||
|
||||
@strawberry.type
|
||||
class GenericJobMutationReturn(MutationReturnInterface):
|
||||
class GenericJobButationReturn(MutationReturnInterface):
|
||||
job: typing.Optional[ApiJob] = None
|
||||
|
|
|
|||
|
|
@ -4,26 +4,18 @@ import typing
|
|||
import strawberry
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||
from selfprivacy_api.jobs import JobStatus
|
||||
|
||||
from traceback import format_tb as format_traceback
|
||||
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobMutationReturn,
|
||||
GenericMutationReturn,
|
||||
)
|
||||
from selfprivacy_api.graphql.common_types.service import (
|
||||
Service,
|
||||
service_to_graphql_service,
|
||||
)
|
||||
|
||||
from selfprivacy_api.actions.services import (
|
||||
move_service,
|
||||
ServiceNotFoundError,
|
||||
VolumeNotFoundError,
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobButationReturn,
|
||||
GenericMutationReturn,
|
||||
)
|
||||
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
from selfprivacy_api.utils.block_devices import BlockDevices
|
||||
|
||||
|
||||
@strawberry.type
|
||||
|
|
@ -42,7 +34,7 @@ class MoveServiceInput:
|
|||
|
||||
|
||||
@strawberry.type
|
||||
class ServiceJobMutationReturn(GenericJobMutationReturn):
|
||||
class ServiceJobMutationReturn(GenericJobButationReturn):
|
||||
"""Service job mutation return type."""
|
||||
|
||||
service: typing.Optional[Service] = None
|
||||
|
|
@ -55,22 +47,14 @@ class ServicesMutations:
|
|||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def enable_service(self, service_id: str) -> ServiceMutationReturn:
|
||||
"""Enable service."""
|
||||
try:
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
return ServiceMutationReturn(
|
||||
success=False,
|
||||
message="Service not found.",
|
||||
code=404,
|
||||
)
|
||||
service.enable()
|
||||
except Exception as e:
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
return ServiceMutationReturn(
|
||||
success=False,
|
||||
message=pretty_error(e),
|
||||
code=400,
|
||||
message="Service not found.",
|
||||
code=404,
|
||||
)
|
||||
|
||||
service.enable()
|
||||
return ServiceMutationReturn(
|
||||
success=True,
|
||||
message="Service enabled.",
|
||||
|
|
@ -81,21 +65,14 @@ class ServicesMutations:
|
|||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def disable_service(self, service_id: str) -> ServiceMutationReturn:
|
||||
"""Disable service."""
|
||||
try:
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
return ServiceMutationReturn(
|
||||
success=False,
|
||||
message="Service not found.",
|
||||
code=404,
|
||||
)
|
||||
service.disable()
|
||||
except Exception as e:
|
||||
service = get_service_by_id(service_id)
|
||||
if service is None:
|
||||
return ServiceMutationReturn(
|
||||
success=False,
|
||||
message=pretty_error(e),
|
||||
code=400,
|
||||
message="Service not found.",
|
||||
code=404,
|
||||
)
|
||||
service.disable()
|
||||
return ServiceMutationReturn(
|
||||
success=True,
|
||||
message="Service disabled.",
|
||||
|
|
@ -160,58 +137,33 @@ class ServicesMutations:
|
|||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def move_service(self, input: MoveServiceInput) -> ServiceJobMutationReturn:
|
||||
"""Move service."""
|
||||
# We need a service instance for a reply later
|
||||
service = get_service_by_id(input.service_id)
|
||||
if service is None:
|
||||
return ServiceJobMutationReturn(
|
||||
success=False,
|
||||
message=f"Service does not exist: {input.service_id}",
|
||||
message="Service not found.",
|
||||
code=404,
|
||||
)
|
||||
|
||||
try:
|
||||
job = move_service(input.service_id, input.location)
|
||||
|
||||
except (ServiceNotFoundError, VolumeNotFoundError) as e:
|
||||
if not service.is_movable():
|
||||
return ServiceJobMutationReturn(
|
||||
success=False,
|
||||
message=pretty_error(e),
|
||||
code=404,
|
||||
)
|
||||
except Exception as e:
|
||||
return ServiceJobMutationReturn(
|
||||
success=False,
|
||||
message=pretty_error(e),
|
||||
message="Service is not movable.",
|
||||
code=400,
|
||||
service=service_to_graphql_service(service),
|
||||
)
|
||||
|
||||
if job.status in [JobStatus.CREATED, JobStatus.RUNNING]:
|
||||
return ServiceJobMutationReturn(
|
||||
success=True,
|
||||
message="Started moving the service.",
|
||||
code=200,
|
||||
service=service_to_graphql_service(service),
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
elif job.status == JobStatus.FINISHED:
|
||||
return ServiceJobMutationReturn(
|
||||
success=True,
|
||||
message="Service moved.",
|
||||
code=200,
|
||||
service=service_to_graphql_service(service),
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
else:
|
||||
volume = BlockDevices().get_block_device(input.location)
|
||||
if volume is None:
|
||||
return ServiceJobMutationReturn(
|
||||
success=False,
|
||||
message=f"While moving service and performing the step '{job.status_text}', error occured: {job.error}",
|
||||
code=400,
|
||||
message="Volume not found.",
|
||||
code=404,
|
||||
service=service_to_graphql_service(service),
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
||||
|
||||
def pretty_error(e: Exception) -> str:
|
||||
traceback = "/r".join(format_traceback(e.__traceback__))
|
||||
return type(e).__name__ + ": " + str(e) + ": " + traceback
|
||||
job = service.move_to_volume(volume)
|
||||
return ServiceJobMutationReturn(
|
||||
success=True,
|
||||
message="Service moved.",
|
||||
code=200,
|
||||
service=service_to_graphql_service(service),
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Users management module"""
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
import strawberry
|
||||
from selfprivacy_api.actions.users import UserNotFound
|
||||
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.actions.ssh import (
|
||||
InvalidPublicKey,
|
||||
KeyAlreadyExists,
|
||||
KeyNotFound,
|
||||
create_ssh_key,
|
||||
remove_ssh_key,
|
||||
)
|
||||
from selfprivacy_api.graphql.common_types.user import (
|
||||
UserMutationReturn,
|
||||
get_user_by_username,
|
||||
)
|
||||
|
||||
|
||||
@strawberry.input
|
||||
class SshMutationInput:
|
||||
"""Input type for ssh mutation"""
|
||||
|
||||
username: str
|
||||
ssh_key: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SshMutations:
|
||||
"""Mutations ssh"""
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def add_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
|
||||
"""Add a new ssh key"""
|
||||
|
||||
try:
|
||||
create_ssh_key(ssh_input.username, ssh_input.ssh_key)
|
||||
except KeyAlreadyExists:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Key already exists",
|
||||
code=409,
|
||||
)
|
||||
except InvalidPublicKey:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
|
||||
code=400,
|
||||
)
|
||||
except UserNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="User not found",
|
||||
code=404,
|
||||
)
|
||||
except Exception as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
|
||||
return UserMutationReturn(
|
||||
success=True,
|
||||
message="New SSH key successfully written",
|
||||
code=201,
|
||||
user=get_user_by_username(ssh_input.username),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def remove_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
|
||||
"""Remove ssh key from user"""
|
||||
|
||||
try:
|
||||
remove_ssh_key(ssh_input.username, ssh_input.ssh_key)
|
||||
except KeyNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Key not found",
|
||||
code=404,
|
||||
)
|
||||
except UserNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="User not found",
|
||||
code=404,
|
||||
)
|
||||
except Exception as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
|
||||
return UserMutationReturn(
|
||||
success=True,
|
||||
message="SSH key successfully removed",
|
||||
code=200,
|
||||
user=get_user_by_username(ssh_input.username),
|
||||
)
|
||||
|
|
@ -4,7 +4,7 @@ from selfprivacy_api.graphql import IsAuthenticated
|
|||
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||
from selfprivacy_api.utils.block_devices import BlockDevices
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobMutationReturn,
|
||||
GenericJobButationReturn,
|
||||
GenericMutationReturn,
|
||||
)
|
||||
from selfprivacy_api.jobs.migrate_to_binds import (
|
||||
|
|
@ -79,10 +79,10 @@ class StorageMutations:
|
|||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def migrate_to_binds(self, input: MigrateToBindsInput) -> GenericJobMutationReturn:
|
||||
def migrate_to_binds(self, input: MigrateToBindsInput) -> GenericJobButationReturn:
|
||||
"""Migrate to binds"""
|
||||
if is_bind_migrated():
|
||||
return GenericJobMutationReturn(
|
||||
return GenericJobButationReturn(
|
||||
success=False, code=409, message="Already migrated to binds"
|
||||
)
|
||||
job = start_bind_migration(
|
||||
|
|
@ -94,7 +94,7 @@ class StorageMutations:
|
|||
pleroma_block_device=input.pleroma_block_device,
|
||||
)
|
||||
)
|
||||
return GenericJobMutationReturn(
|
||||
return GenericJobButationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="Migration to binds started, rebuild the system to apply changes",
|
||||
|
|
|
|||
|
|
@ -3,18 +3,12 @@
|
|||
import typing
|
||||
import strawberry
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobMutationReturn,
|
||||
GenericMutationReturn,
|
||||
MutationReturnInterface,
|
||||
GenericJobMutationReturn,
|
||||
)
|
||||
|
||||
import selfprivacy_api.actions.system as system_actions
|
||||
from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
|
||||
from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage
|
||||
import selfprivacy_api.actions.ssh as ssh_actions
|
||||
|
||||
|
||||
@strawberry.type
|
||||
|
|
@ -32,22 +26,6 @@ class AutoUpgradeSettingsMutationReturn(MutationReturnInterface):
|
|||
allowReboot: bool
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SSHSettingsMutationReturn(MutationReturnInterface):
|
||||
"""A return type for after changing SSH settings"""
|
||||
|
||||
enable: bool
|
||||
password_authentication: bool
|
||||
|
||||
|
||||
@strawberry.input
|
||||
class SSHSettingsInput:
|
||||
"""Input type for SSH settings"""
|
||||
|
||||
enable: bool
|
||||
password_authentication: bool
|
||||
|
||||
|
||||
@strawberry.input
|
||||
class AutoUpgradeSettingsInput:
|
||||
"""Input type for auto upgrade settings"""
|
||||
|
|
@ -99,90 +77,40 @@ class SystemMutations:
|
|||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def change_ssh_settings(
|
||||
self, settings: SSHSettingsInput
|
||||
) -> SSHSettingsMutationReturn:
|
||||
"""Change ssh settings of the server."""
|
||||
ssh_actions.set_ssh_settings(
|
||||
enable=settings.enable,
|
||||
password_authentication=settings.password_authentication,
|
||||
)
|
||||
|
||||
new_settings = ssh_actions.get_ssh_settings()
|
||||
|
||||
return SSHSettingsMutationReturn(
|
||||
def run_system_rebuild(self) -> GenericMutationReturn:
|
||||
system_actions.rebuild_system()
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="SSH settings changed",
|
||||
message="Starting rebuild system",
|
||||
code=200,
|
||||
enable=new_settings.enable,
|
||||
password_authentication=new_settings.passwordAuthentication,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def run_system_rebuild(self) -> GenericJobMutationReturn:
|
||||
try:
|
||||
job = system_actions.rebuild_system()
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
message="Starting system rebuild",
|
||||
code=200,
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
except system_actions.ShellException as e:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def run_system_rollback(self) -> GenericMutationReturn:
|
||||
system_actions.rollback_system()
|
||||
try:
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="Starting system rollback",
|
||||
code=200,
|
||||
)
|
||||
except system_actions.ShellException as e:
|
||||
return GenericMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="Starting rebuild system",
|
||||
code=200,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def run_system_upgrade(self) -> GenericJobMutationReturn:
|
||||
try:
|
||||
job = system_actions.upgrade_system()
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
message="Starting system upgrade",
|
||||
code=200,
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
except system_actions.ShellException as e:
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
def run_system_upgrade(self) -> GenericMutationReturn:
|
||||
system_actions.upgrade_system()
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="Starting rebuild system",
|
||||
code=200,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def reboot_system(self) -> GenericMutationReturn:
|
||||
system_actions.reboot_system()
|
||||
try:
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="System reboot has started",
|
||||
code=200,
|
||||
)
|
||||
except system_actions.ShellException as e:
|
||||
return GenericMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
return GenericMutationReturn(
|
||||
success=True,
|
||||
message="System reboot has started",
|
||||
code=200,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def pull_repository_changes(self) -> GenericMutationReturn:
|
||||
|
|
@ -198,14 +126,3 @@ class SystemMutations:
|
|||
message=f"Failed to pull repository changes:\n{result.data}",
|
||||
code=500,
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def nix_collect_garbage(self) -> GenericJobMutationReturn:
|
||||
job = start_nix_collect_garbage()
|
||||
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="Garbage collector started...",
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,18 +3,10 @@
|
|||
# pylint: disable=too-few-public-methods
|
||||
import strawberry
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.actions.users import UserNotFound
|
||||
from selfprivacy_api.graphql.common_types.user import (
|
||||
UserMutationReturn,
|
||||
get_user_by_username,
|
||||
)
|
||||
from selfprivacy_api.actions.ssh import (
|
||||
InvalidPublicKey,
|
||||
KeyAlreadyExists,
|
||||
KeyNotFound,
|
||||
create_ssh_key,
|
||||
remove_ssh_key,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericMutationReturn,
|
||||
)
|
||||
|
|
@ -29,16 +21,8 @@ class UserMutationInput:
|
|||
password: str
|
||||
|
||||
|
||||
@strawberry.input
|
||||
class SshMutationInput:
|
||||
"""Input type for ssh mutation"""
|
||||
|
||||
username: str
|
||||
ssh_key: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class UsersMutations:
|
||||
class UserMutations:
|
||||
"""Mutations change user settings"""
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
|
|
@ -69,12 +53,6 @@ class UsersMutations:
|
|||
message=str(e),
|
||||
code=400,
|
||||
)
|
||||
except users_actions.InvalidConfiguration as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=400,
|
||||
)
|
||||
except users_actions.UserAlreadyExists as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
|
|
@ -137,73 +115,3 @@ class UsersMutations:
|
|||
code=200,
|
||||
user=get_user_by_username(user.username),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def add_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
|
||||
"""Add a new ssh key"""
|
||||
|
||||
try:
|
||||
create_ssh_key(ssh_input.username, ssh_input.ssh_key)
|
||||
except KeyAlreadyExists:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Key already exists",
|
||||
code=409,
|
||||
)
|
||||
except InvalidPublicKey:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Invalid key type. Only ssh-ed25519, ssh-rsa and ecdsa are supported",
|
||||
code=400,
|
||||
)
|
||||
except UserNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="User not found",
|
||||
code=404,
|
||||
)
|
||||
except Exception as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
|
||||
return UserMutationReturn(
|
||||
success=True,
|
||||
message="New SSH key successfully written",
|
||||
code=201,
|
||||
user=get_user_by_username(ssh_input.username),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def remove_ssh_key(self, ssh_input: SshMutationInput) -> UserMutationReturn:
|
||||
"""Remove ssh key from user"""
|
||||
|
||||
try:
|
||||
remove_ssh_key(ssh_input.username, ssh_input.ssh_key)
|
||||
except KeyNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="Key not found",
|
||||
code=404,
|
||||
)
|
||||
except UserNotFound:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message="User not found",
|
||||
code=404,
|
||||
)
|
||||
except Exception as e:
|
||||
return UserMutationReturn(
|
||||
success=False,
|
||||
message=str(e),
|
||||
code=500,
|
||||
)
|
||||
|
||||
return UserMutationReturn(
|
||||
success=True,
|
||||
message="SSH key successfully removed",
|
||||
code=200,
|
||||
user=get_user_by_username(ssh_input.username),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ import datetime
|
|||
import typing
|
||||
import strawberry
|
||||
from strawberry.types import Info
|
||||
from selfprivacy_api.actions.api_tokens import (
|
||||
get_api_tokens_with_caller_flag,
|
||||
get_api_recovery_token_status,
|
||||
)
|
||||
from selfprivacy_api.actions.api_tokens import get_api_tokens_with_caller_flag
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.utils import parse_date
|
||||
from selfprivacy_api.dependencies import get_api_version as get_api_version_dependency
|
||||
|
||||
from selfprivacy_api.utils.auth import (
|
||||
get_recovery_token_status,
|
||||
is_recovery_token_exists,
|
||||
is_recovery_token_valid,
|
||||
)
|
||||
|
||||
|
||||
def get_api_version() -> str:
|
||||
"""Get API version"""
|
||||
|
|
@ -38,9 +42,17 @@ class ApiRecoveryKeyStatus:
|
|||
|
||||
|
||||
def get_recovery_key_status() -> ApiRecoveryKeyStatus:
|
||||
"""Get recovery key status, times are timezone-aware"""
|
||||
status = get_api_recovery_token_status()
|
||||
if status is None or not status.exists:
|
||||
"""Get recovery key status"""
|
||||
if not is_recovery_token_exists():
|
||||
return ApiRecoveryKeyStatus(
|
||||
exists=False,
|
||||
valid=False,
|
||||
creation_date=None,
|
||||
expiration_date=None,
|
||||
uses_left=None,
|
||||
)
|
||||
status = get_recovery_token_status()
|
||||
if status is None:
|
||||
return ApiRecoveryKeyStatus(
|
||||
exists=False,
|
||||
valid=False,
|
||||
|
|
@ -50,10 +62,12 @@ def get_recovery_key_status() -> ApiRecoveryKeyStatus:
|
|||
)
|
||||
return ApiRecoveryKeyStatus(
|
||||
exists=True,
|
||||
valid=status.valid,
|
||||
creation_date=status.date,
|
||||
expiration_date=status.expiration,
|
||||
uses_left=status.uses_left,
|
||||
valid=is_recovery_token_valid(),
|
||||
creation_date=parse_date(status["date"]),
|
||||
expiration_date=parse_date(status["expiration"])
|
||||
if status["expiration"] is not None
|
||||
else None,
|
||||
uses_left=status["uses_left"] if status["uses_left"] is not None else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
"""Backup"""
|
||||
# pylint: disable=too-few-public-methods
|
||||
import typing
|
||||
import strawberry
|
||||
|
||||
|
||||
from selfprivacy_api.backup import Backups
|
||||
from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
||||
from selfprivacy_api.graphql.queries.providers import BackupProvider
|
||||
from selfprivacy_api.graphql.common_types.service import (
|
||||
Service,
|
||||
ServiceStatusEnum,
|
||||
SnapshotInfo,
|
||||
service_to_graphql_service,
|
||||
)
|
||||
from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas
|
||||
from selfprivacy_api.services import get_service_by_id
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class BackupConfiguration:
|
||||
provider: BackupProvider
|
||||
# When server is lost, the app should have the key to decrypt backups
|
||||
# on a new server
|
||||
encryption_key: str
|
||||
# False when repo is not initialized and not ready to be used
|
||||
is_initialized: bool
|
||||
# If none, autobackups are disabled
|
||||
autobackup_period: typing.Optional[int]
|
||||
# None is equal to all quotas being unlimited (-1). Optional for compatibility reasons.
|
||||
autobackup_quotas: AutobackupQuotas
|
||||
# Bucket name for Backblaze, path for some other providers
|
||||
location_name: typing.Optional[str]
|
||||
location_id: typing.Optional[str]
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Backup:
|
||||
@strawberry.field
|
||||
def configuration(self) -> BackupConfiguration:
|
||||
return BackupConfiguration(
|
||||
provider=Backups.provider().name,
|
||||
encryption_key=LocalBackupSecret.get(),
|
||||
is_initialized=Backups.is_initted(),
|
||||
autobackup_period=Backups.autobackup_period_minutes(),
|
||||
location_name=Backups.provider().location,
|
||||
location_id=Backups.provider().repo_id,
|
||||
autobackup_quotas=Backups.autobackup_quotas(),
|
||||
)
|
||||
|
||||
@strawberry.field
|
||||
def all_snapshots(self) -> typing.List[SnapshotInfo]:
|
||||
if not Backups.is_initted():
|
||||
return []
|
||||
result = []
|
||||
snapshots = Backups.get_all_snapshots()
|
||||
for snap in snapshots:
|
||||
service = get_service_by_id(snap.service_name)
|
||||
if service is None:
|
||||
service = Service(
|
||||
id=snap.service_name,
|
||||
display_name=f"{snap.service_name} (Orphaned)",
|
||||
description="",
|
||||
svg_icon="",
|
||||
is_movable=False,
|
||||
is_required=False,
|
||||
is_enabled=False,
|
||||
status=ServiceStatusEnum.OFF,
|
||||
url=None,
|
||||
dns_records=None,
|
||||
can_be_backed_up=False,
|
||||
backup_description="",
|
||||
)
|
||||
else:
|
||||
service = service_to_graphql_service(service)
|
||||
graphql_snap = SnapshotInfo(
|
||||
id=snap.id,
|
||||
service=service,
|
||||
created_at=snap.created_at,
|
||||
reason=snap.reason,
|
||||
)
|
||||
result.append(graphql_snap)
|
||||
return result
|
||||
|
|
@ -15,9 +15,10 @@ from selfprivacy_api.jobs import Jobs
|
|||
class Job:
|
||||
@strawberry.field
|
||||
def get_jobs(self) -> typing.List[ApiJob]:
|
||||
Jobs.get_jobs()
|
||||
|
||||
return [job_to_api_job(job) for job in Jobs.get_jobs()]
|
||||
Jobs.get_instance().get_jobs()
|
||||
|
||||
return [job_to_api_job(job) for job in Jobs.get_instance().get_jobs()]
|
||||
|
||||
@strawberry.field
|
||||
def get_job(self, job_id: str) -> typing.Optional[ApiJob]:
|
||||
|
|
|
|||
|
|
@ -6,20 +6,8 @@ import strawberry
|
|||
@strawberry.enum
|
||||
class DnsProvider(Enum):
|
||||
CLOUDFLARE = "CLOUDFLARE"
|
||||
DIGITALOCEAN = "DIGITALOCEAN"
|
||||
DESEC = "DESEC"
|
||||
|
||||
|
||||
@strawberry.enum
|
||||
class ServerProvider(Enum):
|
||||
HETZNER = "HETZNER"
|
||||
DIGITALOCEAN = "DIGITALOCEAN"
|
||||
|
||||
|
||||
@strawberry.enum
|
||||
class BackupProvider(Enum):
|
||||
BACKBLAZE = "BACKBLAZE"
|
||||
NONE = "NONE"
|
||||
# for testing purposes, make sure not selectable in prod.
|
||||
MEMORY = "MEMORY"
|
||||
FILE = "FILE"
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class Storage:
|
|||
else str(volume.size),
|
||||
free_space=str(volume.fsavail),
|
||||
used_space=str(volume.fsused),
|
||||
root=volume.is_root(),
|
||||
root=volume.name == "sda1",
|
||||
name=volume.name,
|
||||
model=volume.model,
|
||||
serial=volume.serial,
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ class SystemDomainInfo:
|
|||
content=record.content,
|
||||
ttl=record.ttl,
|
||||
priority=record.priority,
|
||||
display_name=record.display_name,
|
||||
)
|
||||
for record in get_all_required_dns_records()
|
||||
]
|
||||
|
|
@ -45,7 +44,7 @@ def get_system_domain_info() -> SystemDomainInfo:
|
|||
return SystemDomainInfo(
|
||||
domain=user_data["domain"],
|
||||
hostname=user_data["hostname"],
|
||||
provider=user_data["dns"]["provider"],
|
||||
provider=DnsProvider.CLOUDFLARE,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -134,11 +133,7 @@ class SystemProviderInfo:
|
|||
|
||||
def get_system_provider_info() -> SystemProviderInfo:
|
||||
"""Get system provider info"""
|
||||
with ReadUserData() as user_data:
|
||||
return SystemProviderInfo(
|
||||
provider=user_data["server"]["provider"],
|
||||
id="UNKNOWN",
|
||||
)
|
||||
return SystemProviderInfo(provider=ServerProvider.HETZNER, id="UNKNOWN")
|
||||
|
||||
|
||||
@strawberry.type
|
||||
|
|
|
|||
|
|
@ -5,30 +5,21 @@ import asyncio
|
|||
from typing import AsyncGenerator
|
||||
import strawberry
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
from selfprivacy_api.graphql.mutations.deprecated_mutations import (
|
||||
DeprecatedApiMutations,
|
||||
DeprecatedJobMutations,
|
||||
DeprecatedServicesMutations,
|
||||
DeprecatedStorageMutations,
|
||||
DeprecatedSystemMutations,
|
||||
DeprecatedUsersMutations,
|
||||
)
|
||||
from selfprivacy_api.graphql.mutations.api_mutations import ApiMutations
|
||||
from selfprivacy_api.graphql.mutations.job_mutations import JobMutations
|
||||
from selfprivacy_api.graphql.mutations.mutation_interface import GenericMutationReturn
|
||||
from selfprivacy_api.graphql.mutations.services_mutations import ServicesMutations
|
||||
from selfprivacy_api.graphql.mutations.ssh_mutations import SshMutations
|
||||
from selfprivacy_api.graphql.mutations.storage_mutations import StorageMutations
|
||||
from selfprivacy_api.graphql.mutations.system_mutations import SystemMutations
|
||||
from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
|
||||
|
||||
from selfprivacy_api.graphql.queries.api_queries import Api
|
||||
from selfprivacy_api.graphql.queries.backup import Backup
|
||||
from selfprivacy_api.graphql.queries.jobs import Job
|
||||
from selfprivacy_api.graphql.queries.services import Services
|
||||
from selfprivacy_api.graphql.queries.storage import Storage
|
||||
from selfprivacy_api.graphql.queries.system import System
|
||||
|
||||
from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
|
||||
from selfprivacy_api.graphql.mutations.users_mutations import UserMutations
|
||||
from selfprivacy_api.graphql.queries.users import Users
|
||||
from selfprivacy_api.jobs.test import test_job
|
||||
|
||||
|
|
@ -37,16 +28,16 @@ from selfprivacy_api.jobs.test import test_job
|
|||
class Query:
|
||||
"""Root schema for queries"""
|
||||
|
||||
@strawberry.field
|
||||
def api(self) -> Api:
|
||||
"""API access status"""
|
||||
return Api()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def system(self) -> System:
|
||||
"""System queries"""
|
||||
return System()
|
||||
|
||||
@strawberry.field
|
||||
def api(self) -> Api:
|
||||
"""API access status"""
|
||||
return Api()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def users(self) -> Users:
|
||||
"""Users queries"""
|
||||
|
|
@ -67,58 +58,19 @@ class Query:
|
|||
"""Services queries"""
|
||||
return Services()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def backup(self) -> Backup:
|
||||
"""Backup queries"""
|
||||
return Backup()
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Mutation(
|
||||
DeprecatedApiMutations,
|
||||
DeprecatedSystemMutations,
|
||||
DeprecatedUsersMutations,
|
||||
DeprecatedStorageMutations,
|
||||
DeprecatedServicesMutations,
|
||||
DeprecatedJobMutations,
|
||||
ApiMutations,
|
||||
SystemMutations,
|
||||
UserMutations,
|
||||
SshMutations,
|
||||
StorageMutations,
|
||||
ServicesMutations,
|
||||
JobMutations,
|
||||
):
|
||||
"""Root schema for mutations"""
|
||||
|
||||
@strawberry.field
|
||||
def api(self) -> ApiMutations:
|
||||
"""API mutations"""
|
||||
return ApiMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def system(self) -> SystemMutations:
|
||||
"""System mutations"""
|
||||
return SystemMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def users(self) -> UsersMutations:
|
||||
"""Users mutations"""
|
||||
return UsersMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def storage(self) -> StorageMutations:
|
||||
"""Storage mutations"""
|
||||
return StorageMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def services(self) -> ServicesMutations:
|
||||
"""Services mutations"""
|
||||
return ServicesMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def jobs(self) -> JobMutations:
|
||||
"""Jobs mutations"""
|
||||
return JobMutations()
|
||||
|
||||
@strawberry.field(permission_classes=[IsAuthenticated])
|
||||
def backup(self) -> BackupMutations:
|
||||
"""Backup mutations"""
|
||||
return BackupMutations()
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def test_mutation(self) -> GenericMutationReturn:
|
||||
"""Test mutation"""
|
||||
|
|
@ -143,8 +95,4 @@ class Subscription:
|
|||
await asyncio.sleep(0.5)
|
||||
|
||||
|
||||
schema = strawberry.Schema(
|
||||
query=Query,
|
||||
mutation=Mutation,
|
||||
subscription=Subscription,
|
||||
)
|
||||
schema = strawberry.Schema(query=Query, mutation=Mutation, subscription=Subscription)
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ A job is a dictionary with the following keys:
|
|||
- name: name of the job
|
||||
- description: description of the job
|
||||
- status: status of the job
|
||||
- created_at: date of creation of the job, naive localtime
|
||||
- updated_at: date of last update of the job, naive localtime
|
||||
- created_at: date of creation of the job
|
||||
- updated_at: date of last update of the job
|
||||
- finished_at: date of finish of the job
|
||||
- error: error message if the job failed
|
||||
- result: result of the job
|
||||
|
|
@ -17,20 +17,19 @@ A job is a dictionary with the following keys:
|
|||
import typing
|
||||
import datetime
|
||||
from uuid import UUID
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from selfprivacy_api.utils.redis_pool import RedisPool
|
||||
|
||||
JOB_EXPIRATION_SECONDS = 10 * 24 * 60 * 60 # ten days
|
||||
|
||||
STATUS_LOGS_PREFIX = "jobs_logs:status:"
|
||||
PROGRESS_LOGS_PREFIX = "jobs_logs:progress:"
|
||||
from selfprivacy_api.utils import ReadUserData, UserDataFiles, WriteUserData
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
class JobStatus(Enum):
|
||||
"""
|
||||
Status of a job.
|
||||
"""
|
||||
|
|
@ -65,15 +64,36 @@ class Jobs:
|
|||
Jobs class.
|
||||
"""
|
||||
|
||||
__instance = None
|
||||
|
||||
@staticmethod
|
||||
def get_instance():
|
||||
"""
|
||||
Singleton method.
|
||||
"""
|
||||
if Jobs.__instance is None:
|
||||
Jobs()
|
||||
if Jobs.__instance is None:
|
||||
raise Exception("Couldn't init Jobs singleton!")
|
||||
return Jobs.__instance
|
||||
return Jobs.__instance
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Initialize the jobs list.
|
||||
"""
|
||||
if Jobs.__instance is not None:
|
||||
raise Exception("This class is a singleton!")
|
||||
else:
|
||||
Jobs.__instance = self
|
||||
|
||||
@staticmethod
|
||||
def reset() -> None:
|
||||
"""
|
||||
Reset the jobs list.
|
||||
"""
|
||||
jobs = Jobs.get_jobs()
|
||||
for job in jobs:
|
||||
Jobs.remove(job)
|
||||
Jobs.reset_logs()
|
||||
with WriteUserData(UserDataFiles.JOBS) as user_data:
|
||||
user_data["jobs"] = []
|
||||
|
||||
@staticmethod
|
||||
def add(
|
||||
|
|
@ -101,83 +121,34 @@ class Jobs:
|
|||
error=None,
|
||||
result=None,
|
||||
)
|
||||
redis = RedisPool().get_connection()
|
||||
_store_job_as_hash(redis, _redis_key_from_uuid(job.uid), job)
|
||||
with WriteUserData(UserDataFiles.JOBS) as user_data:
|
||||
try:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
user_data["jobs"].append(json.loads(job.json()))
|
||||
except json.decoder.JSONDecodeError:
|
||||
user_data["jobs"] = [json.loads(job.json())]
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def remove(job: Job) -> None:
|
||||
def remove(self, job: Job) -> None:
|
||||
"""
|
||||
Remove a job from the jobs list.
|
||||
"""
|
||||
Jobs.remove_by_uid(str(job.uid))
|
||||
self.remove_by_uid(str(job.uid))
|
||||
|
||||
@staticmethod
|
||||
def remove_by_uid(job_uuid: str) -> bool:
|
||||
def remove_by_uid(self, job_uuid: str) -> bool:
|
||||
"""
|
||||
Remove a job from the jobs list.
|
||||
"""
|
||||
redis = RedisPool().get_connection()
|
||||
key = _redis_key_from_uuid(job_uuid)
|
||||
if redis.exists(key):
|
||||
redis.delete(key)
|
||||
return True
|
||||
with WriteUserData(UserDataFiles.JOBS) as user_data:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
for i, j in enumerate(user_data["jobs"]):
|
||||
if j["uid"] == job_uuid:
|
||||
del user_data["jobs"][i]
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def reset_logs() -> None:
|
||||
redis = RedisPool().get_connection()
|
||||
for key in redis.keys(STATUS_LOGS_PREFIX + "*"):
|
||||
redis.delete(key)
|
||||
|
||||
@staticmethod
|
||||
def log_status_update(job: Job, status: JobStatus) -> None:
|
||||
redis = RedisPool().get_connection()
|
||||
key = _status_log_key_from_uuid(job.uid)
|
||||
redis.lpush(key, status.value)
|
||||
redis.expire(key, 10)
|
||||
|
||||
@staticmethod
|
||||
def log_progress_update(job: Job, progress: int) -> None:
|
||||
redis = RedisPool().get_connection()
|
||||
key = _progress_log_key_from_uuid(job.uid)
|
||||
redis.lpush(key, progress)
|
||||
redis.expire(key, 10)
|
||||
|
||||
@staticmethod
|
||||
def status_updates(job: Job) -> list[JobStatus]:
|
||||
result: list[JobStatus] = []
|
||||
|
||||
redis = RedisPool().get_connection()
|
||||
key = _status_log_key_from_uuid(job.uid)
|
||||
if not redis.exists(key):
|
||||
return []
|
||||
|
||||
status_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore
|
||||
for status in status_strings:
|
||||
try:
|
||||
result.append(JobStatus[status])
|
||||
except KeyError as error:
|
||||
raise ValueError("impossible job status: " + status) from error
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def progress_updates(job: Job) -> list[int]:
|
||||
result: list[int] = []
|
||||
|
||||
redis = RedisPool().get_connection()
|
||||
key = _progress_log_key_from_uuid(job.uid)
|
||||
if not redis.exists(key):
|
||||
return []
|
||||
|
||||
progress_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore
|
||||
for progress in progress_strings:
|
||||
try:
|
||||
result.append(int(progress))
|
||||
except KeyError as error:
|
||||
raise ValueError("impossible job progress: " + progress) from error
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def update(
|
||||
job: Job,
|
||||
|
|
@ -198,49 +169,36 @@ class Jobs:
|
|||
job.description = description
|
||||
if status_text is not None:
|
||||
job.status_text = status_text
|
||||
|
||||
# if it is finished it is 100
|
||||
# unless user says otherwise
|
||||
if status == JobStatus.FINISHED and progress is None:
|
||||
progress = 100
|
||||
if progress is not None and job.progress != progress:
|
||||
if progress is not None:
|
||||
job.progress = progress
|
||||
Jobs.log_progress_update(job, progress)
|
||||
|
||||
job.status = status
|
||||
Jobs.log_status_update(job, status)
|
||||
job.updated_at = datetime.datetime.now()
|
||||
job.error = error
|
||||
job.result = result
|
||||
if status in (JobStatus.FINISHED, JobStatus.ERROR):
|
||||
job.finished_at = datetime.datetime.now()
|
||||
|
||||
redis = RedisPool().get_connection()
|
||||
key = _redis_key_from_uuid(job.uid)
|
||||
if redis.exists(key):
|
||||
_store_job_as_hash(redis, key, job)
|
||||
if status in (JobStatus.FINISHED, JobStatus.ERROR):
|
||||
redis.expire(key, JOB_EXPIRATION_SECONDS)
|
||||
with WriteUserData(UserDataFiles.JOBS) as user_data:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
for i, j in enumerate(user_data["jobs"]):
|
||||
if j["uid"] == str(job.uid):
|
||||
user_data["jobs"][i] = json.loads(job.json())
|
||||
break
|
||||
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def set_expiration(job: Job, expiration_seconds: int) -> Job:
|
||||
redis = RedisPool().get_connection()
|
||||
key = _redis_key_from_uuid(job.uid)
|
||||
if redis.exists(key):
|
||||
redis.expire(key, expiration_seconds)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def get_job(uid: str) -> typing.Optional[Job]:
|
||||
"""
|
||||
Get a job from the jobs list.
|
||||
"""
|
||||
redis = RedisPool().get_connection()
|
||||
key = _redis_key_from_uuid(uid)
|
||||
if redis.exists(key):
|
||||
return _job_from_hash(redis, key)
|
||||
with ReadUserData(UserDataFiles.JOBS) as user_data:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
for job in user_data["jobs"]:
|
||||
if job["uid"] == uid:
|
||||
return Job(**job)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -248,76 +206,23 @@ class Jobs:
|
|||
"""
|
||||
Get the jobs list.
|
||||
"""
|
||||
redis = RedisPool().get_connection()
|
||||
job_keys = redis.keys("jobs:*")
|
||||
jobs = []
|
||||
for job_key in job_keys:
|
||||
job = _job_from_hash(redis, job_key)
|
||||
if job is not None:
|
||||
jobs.append(job)
|
||||
return jobs
|
||||
with ReadUserData(UserDataFiles.JOBS) as user_data:
|
||||
try:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
return [Job(**job) for job in user_data["jobs"]]
|
||||
except json.decoder.JSONDecodeError:
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def is_busy() -> bool:
|
||||
"""
|
||||
Check if there is a job running.
|
||||
"""
|
||||
for job in Jobs.get_jobs():
|
||||
if job.status == JobStatus.RUNNING:
|
||||
return True
|
||||
with ReadUserData(UserDataFiles.JOBS) as user_data:
|
||||
if "jobs" not in user_data:
|
||||
user_data["jobs"] = []
|
||||
for job in user_data["jobs"]:
|
||||
if job["status"] == JobStatus.RUNNING.value:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def report_progress(progress: int, job: Job, status_text: str) -> None:
|
||||
"""
|
||||
A terse way to call a common operation, for readability
|
||||
job.report_progress() would be even better
|
||||
but it would go against how this file is written
|
||||
"""
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text=status_text,
|
||||
progress=progress,
|
||||
)
|
||||
|
||||
|
||||
def _redis_key_from_uuid(uuid_string) -> str:
|
||||
return "jobs:" + str(uuid_string)
|
||||
|
||||
|
||||
def _status_log_key_from_uuid(uuid_string) -> str:
|
||||
return STATUS_LOGS_PREFIX + str(uuid_string)
|
||||
|
||||
|
||||
def _progress_log_key_from_uuid(uuid_string) -> str:
|
||||
return PROGRESS_LOGS_PREFIX + str(uuid_string)
|
||||
|
||||
|
||||
def _store_job_as_hash(redis, redis_key, model) -> None:
|
||||
for key, value in model.dict().items():
|
||||
if isinstance(value, uuid.UUID):
|
||||
value = str(value)
|
||||
if isinstance(value, datetime.datetime):
|
||||
value = value.isoformat()
|
||||
if isinstance(value, JobStatus):
|
||||
value = value.value
|
||||
redis.hset(redis_key, key, str(value))
|
||||
|
||||
|
||||
def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
|
||||
if redis.exists(redis_key):
|
||||
job_dict = redis.hgetall(redis_key)
|
||||
for date in [
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"finished_at",
|
||||
]:
|
||||
if job_dict[date] != "None":
|
||||
job_dict[date] = datetime.datetime.fromisoformat(job_dict[date])
|
||||
for key in job_dict.keys():
|
||||
if job_dict[key] == "None":
|
||||
job_dict[key] = None
|
||||
|
||||
return Job(**job_dict)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@ def move_folder(
|
|||
|
||||
try:
|
||||
data_path.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||
except Exception as error:
|
||||
print(f"Error creating data path: {error}")
|
||||
except Exception as e:
|
||||
print(f"Error creating data path: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,147 +0,0 @@
|
|||
import re
|
||||
import subprocess
|
||||
from typing import Tuple, Iterable
|
||||
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
|
||||
from selfprivacy_api.jobs import JobStatus, Jobs, Job
|
||||
|
||||
|
||||
class ShellException(Exception):
|
||||
"""Shell-related errors"""
|
||||
|
||||
|
||||
COMPLETED_WITH_ERROR = "Error occurred, please report this to the support chat."
|
||||
RESULT_WAS_NOT_FOUND_ERROR = (
|
||||
"We are sorry, garbage collection result was not found. "
|
||||
"Something went wrong, please report this to the support chat."
|
||||
)
|
||||
CLEAR_COMPLETED = "Garbage collection completed."
|
||||
|
||||
|
||||
def delete_old_gens_and_return_dead_report() -> str:
|
||||
subprocess.run(
|
||||
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
return " " if result is None else result
|
||||
|
||||
|
||||
def run_nix_collect_garbage() -> Iterable[bytes]:
|
||||
process = subprocess.Popen(
|
||||
["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
return process.stdout if process.stdout else iter([])
|
||||
|
||||
|
||||
def parse_line(job: Job, line: str) -> Job:
|
||||
"""
|
||||
We parse the string for the presence of a final line,
|
||||
with the final amount of space cleared.
|
||||
Simply put, we're just looking for a similar string:
|
||||
"1537 store paths deleted, 339.84 MiB freed".
|
||||
"""
|
||||
pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
|
||||
match = re.search(pattern, line)
|
||||
|
||||
if match is None:
|
||||
raise ShellException("nix returned gibberish output")
|
||||
|
||||
else:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
status_text=CLEAR_COMPLETED,
|
||||
result=f"{match.group(0)} have been cleared",
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
|
||||
completed_packages = 0
|
||||
prev_progress = 0
|
||||
|
||||
for line in stream:
|
||||
line = line.decode("utf-8")
|
||||
|
||||
if "deleting '/nix/store/" in line:
|
||||
completed_packages += 1
|
||||
percent = int((completed_packages / total_dead_packages) * 100)
|
||||
|
||||
if percent - prev_progress >= 5:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
progress=percent,
|
||||
status_text="Cleaning...",
|
||||
)
|
||||
prev_progress = percent
|
||||
|
||||
elif "store paths deleted," in line:
|
||||
parse_line(job, line)
|
||||
|
||||
|
||||
def get_dead_packages(output) -> Tuple[int, float]:
|
||||
dead = len(re.findall("/nix/store/", output))
|
||||
percent = 0
|
||||
if dead != 0:
|
||||
percent = 100 / dead
|
||||
return dead, percent
|
||||
|
||||
|
||||
@huey.task()
|
||||
def calculate_and_clear_dead_paths(job: Job):
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
progress=0,
|
||||
status_text="Calculate the number of dead packages...",
|
||||
)
|
||||
|
||||
dead_packages, package_equal_to_percent = get_dead_packages(
|
||||
delete_old_gens_and_return_dead_report()
|
||||
)
|
||||
|
||||
if dead_packages == 0:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
status_text="Nothing to clear",
|
||||
result="System is clear",
|
||||
)
|
||||
return True
|
||||
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
progress=0,
|
||||
status_text=f"Found {dead_packages} packages to remove!",
|
||||
)
|
||||
|
||||
stream = run_nix_collect_garbage()
|
||||
try:
|
||||
process_stream(job, stream, dead_packages)
|
||||
except ShellException as error:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
status_text=COMPLETED_WITH_ERROR,
|
||||
error=RESULT_WAS_NOT_FOUND_ERROR,
|
||||
)
|
||||
|
||||
|
||||
def start_nix_collect_garbage() -> Job:
|
||||
job = Jobs.add(
|
||||
type_id="maintenance.collect_nix_garbage",
|
||||
name="Collect garbage",
|
||||
description="Cleaning up unused packages",
|
||||
)
|
||||
|
||||
calculate_and_clear_dead_paths(job=job)
|
||||
|
||||
return job
|
||||
|
|
@ -5,7 +5,7 @@ from selfprivacy_api.jobs import JobStatus, Jobs
|
|||
|
||||
@huey.task()
|
||||
def test_job():
|
||||
job = Jobs.add(
|
||||
job = Jobs.get_instance().add(
|
||||
type_id="test",
|
||||
name="Test job",
|
||||
description="This is a test job.",
|
||||
|
|
@ -14,42 +14,42 @@ def test_job():
|
|||
progress=0,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Performing pre-move checks...",
|
||||
progress=5,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Performing pre-move checks...",
|
||||
progress=10,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Performing pre-move checks...",
|
||||
progress=15,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Performing pre-move checks...",
|
||||
progress=20,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Performing pre-move checks...",
|
||||
progress=25,
|
||||
)
|
||||
time.sleep(5)
|
||||
Jobs.update(
|
||||
Jobs.get_instance().update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
status_text="Job finished.",
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
"""
|
||||
A task to start the system upgrade or rebuild by starting a systemd unit.
|
||||
After starting, track the status of the systemd unit and update the Job
|
||||
status accordingly.
|
||||
"""
|
||||
import subprocess
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
from selfprivacy_api.jobs import JobStatus, Jobs, Job
|
||||
from selfprivacy_api.utils.waitloop import wait_until_true
|
||||
from selfprivacy_api.utils.systemd import (
|
||||
get_service_status,
|
||||
get_last_log_lines,
|
||||
ServiceStatus,
|
||||
)
|
||||
|
||||
START_TIMEOUT = 60 * 5
|
||||
START_INTERVAL = 1
|
||||
RUN_TIMEOUT = 60 * 60
|
||||
RUN_INTERVAL = 5
|
||||
|
||||
|
||||
def check_if_started(unit_name: str):
|
||||
"""Check if the systemd unit has started"""
|
||||
try:
|
||||
status = get_service_status(unit_name)
|
||||
if status == ServiceStatus.ACTIVE:
|
||||
return True
|
||||
return False
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def check_running_status(job: Job, unit_name: str):
|
||||
"""Check if the systemd unit is running"""
|
||||
try:
|
||||
status = get_service_status(unit_name)
|
||||
if status == ServiceStatus.INACTIVE:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
result="System rebuilt.",
|
||||
progress=100,
|
||||
)
|
||||
return True
|
||||
if status == ServiceStatus.FAILED:
|
||||
log_lines = get_last_log_lines(unit_name, 10)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
error="System rebuild failed. Last log lines:\n" + "\n".join(log_lines),
|
||||
)
|
||||
return True
|
||||
if status == ServiceStatus.ACTIVE:
|
||||
log_lines = get_last_log_lines(unit_name, 1)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text=log_lines[0] if len(log_lines) > 0 else "",
|
||||
)
|
||||
return False
|
||||
return False
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def rebuild_system(job: Job, upgrade: bool = False):
|
||||
"""
|
||||
Broken out to allow calling it synchronously.
|
||||
We cannot just block until task is done because it will require a second worker
|
||||
Which we do not have
|
||||
"""
|
||||
|
||||
unit_name = "sp-nixos-upgrade.service" if upgrade else "sp-nixos-rebuild.service"
|
||||
try:
|
||||
command = ["systemctl", "start", unit_name]
|
||||
subprocess.run(
|
||||
command,
|
||||
check=True,
|
||||
start_new_session=True,
|
||||
shell=False,
|
||||
)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Starting the system rebuild...",
|
||||
)
|
||||
# Wait for the systemd unit to start
|
||||
try:
|
||||
wait_until_true(
|
||||
lambda: check_if_started(unit_name),
|
||||
timeout_sec=START_TIMEOUT,
|
||||
interval=START_INTERVAL,
|
||||
)
|
||||
except TimeoutError:
|
||||
log_lines = get_last_log_lines(unit_name, 10)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
error="System rebuild timed out. Last log lines:\n"
|
||||
+ "\n".join(log_lines),
|
||||
)
|
||||
return
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.RUNNING,
|
||||
status_text="Rebuilding the system...",
|
||||
)
|
||||
# Wait for the systemd unit to finish
|
||||
try:
|
||||
wait_until_true(
|
||||
lambda: check_running_status(job, unit_name),
|
||||
timeout_sec=RUN_TIMEOUT,
|
||||
interval=RUN_INTERVAL,
|
||||
)
|
||||
except TimeoutError:
|
||||
log_lines = get_last_log_lines(unit_name, 10)
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
error="System rebuild timed out. Last log lines:\n"
|
||||
+ "\n".join(log_lines),
|
||||
)
|
||||
return
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.ERROR,
|
||||
status_text=str(e),
|
||||
)
|
||||
|
||||
|
||||
@huey.task()
|
||||
def rebuild_system_task(job: Job, upgrade: bool = False):
|
||||
"""Rebuild the system"""
|
||||
rebuild_system(job, upgrade)
|
||||
|
|
@ -8,16 +8,21 @@ at api.skippedMigrations in userdata.json and populating it
|
|||
with IDs of the migrations to skip.
|
||||
Adding DISABLE_ALL to that array disables the migrations module entirely.
|
||||
"""
|
||||
|
||||
from selfprivacy_api.utils import ReadUserData, UserDataFiles
|
||||
from selfprivacy_api.migrations.write_token_to_redis import WriteTokenToRedis
|
||||
from selfprivacy_api.migrations.check_for_system_rebuild_jobs import (
|
||||
CheckForSystemRebuildJobs,
|
||||
from selfprivacy_api.migrations.check_for_failed_binds_migration import CheckForFailedBindsMigration
|
||||
from selfprivacy_api.utils import ReadUserData
|
||||
from selfprivacy_api.migrations.fix_nixos_config_branch import FixNixosConfigBranch
|
||||
from selfprivacy_api.migrations.create_tokens_json import CreateTokensJson
|
||||
from selfprivacy_api.migrations.migrate_to_selfprivacy_channel import (
|
||||
MigrateToSelfprivacyChannel,
|
||||
)
|
||||
from selfprivacy_api.migrations.mount_volume import MountVolume
|
||||
|
||||
migrations = [
|
||||
WriteTokenToRedis(),
|
||||
CheckForSystemRebuildJobs(),
|
||||
FixNixosConfigBranch(),
|
||||
CreateTokensJson(),
|
||||
MigrateToSelfprivacyChannel(),
|
||||
MountVolume(),
|
||||
CheckForFailedBindsMigration(),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -26,7 +31,7 @@ def run_migrations():
|
|||
Go over all migrations. If they are not skipped in userdata file, run them
|
||||
if the migration needed.
|
||||
"""
|
||||
with ReadUserData(UserDataFiles.SECRETS) as data:
|
||||
with ReadUserData() as data:
|
||||
if "api" not in data:
|
||||
skipped_migrations = []
|
||||
elif "skippedMigrations" not in data["api"]:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
from selfprivacy_api.jobs import JobStatus, Jobs
|
||||
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
from selfprivacy_api.utils import WriteUserData
|
||||
|
||||
|
||||
class CheckForFailedBindsMigration(Migration):
|
||||
"""Mount volume."""
|
||||
|
||||
def get_migration_name(self):
|
||||
return "check_for_failed_binds_migration"
|
||||
|
||||
def get_migration_description(self):
|
||||
return "If binds migration failed, try again."
|
||||
|
||||
def is_migration_needed(self):
|
||||
try:
|
||||
jobs = Jobs.get_instance().get_jobs()
|
||||
# If there is a job with type_id "migrations.migrate_to_binds" and status is not "FINISHED",
|
||||
# then migration is needed and job is deleted
|
||||
for job in jobs:
|
||||
if (
|
||||
job.type_id == "migrations.migrate_to_binds"
|
||||
and job.status != JobStatus.FINISHED
|
||||
):
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
def migrate(self):
|
||||
# Get info about existing volumes
|
||||
# Write info about volumes to userdata.json
|
||||
try:
|
||||
jobs = Jobs.get_instance().get_jobs()
|
||||
for job in jobs:
|
||||
if (
|
||||
job.type_id == "migrations.migrate_to_binds"
|
||||
and job.status != JobStatus.FINISHED
|
||||
):
|
||||
Jobs.get_instance().remove(job)
|
||||
with WriteUserData() as userdata:
|
||||
userdata["useBinds"] = False
|
||||
print("Done")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("Error mounting volume")
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
from selfprivacy_api.migrations.migration import Migration
|
||||
from selfprivacy_api.jobs import JobStatus, Jobs
|
||||
|
||||
|
||||
class CheckForSystemRebuildJobs(Migration):
|
||||
"""Check if there are unfinished system rebuild jobs and finish them"""
|
||||
|
||||
def get_migration_name(self):
|
||||
return "check_for_system_rebuild_jobs"
|
||||
|
||||
def get_migration_description(self):
|
||||
return "Check if there are unfinished system rebuild jobs and finish them"
|
||||
|
||||
def is_migration_needed(self):
|
||||
# Check if there are any unfinished system rebuild jobs
|
||||
for job in Jobs.get_jobs():
|
||||
if (
|
||||
job.type_id
|
||||
in [
|
||||
"system.nixos.rebuild",
|
||||
"system.nixos.upgrade",
|
||||
]
|
||||
) and job.status in [
|
||||
JobStatus.CREATED,
|
||||
JobStatus.RUNNING,
|
||||
]:
|
||||
return True
|
||||
|
||||
def migrate(self):
|
||||
# As the API is restarted, we assume that the jobs are finished
|
||||
for job in Jobs.get_jobs():
|
||||
if (
|
||||
job.type_id
|
||||
in [
|
||||
"system.nixos.rebuild",
|
||||
"system.nixos.upgrade",
|
||||
]
|
||||
) and job.status in [
|
||||
JobStatus.CREATED,
|
||||
JobStatus.RUNNING,
|
||||
]:
|
||||
Jobs.update(
|
||||
job=job,
|
||||
status=JobStatus.FINISHED,
|
||||
result="System rebuilt.",
|
||||
progress=100,
|
||||
)
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
from datetime import datetime
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
from selfprivacy_api.utils import TOKENS_FILE, ReadUserData
|
||||
|
||||
|
||||
class CreateTokensJson(Migration):
|
||||
def get_migration_name(self):
|
||||
return "create_tokens_json"
|
||||
|
||||
def get_migration_description(self):
|
||||
return """Selfprivacy API used a single token in userdata.json for authentication.
|
||||
This migration creates a new tokens.json file with the old token in it.
|
||||
This migration runs if the tokens.json file does not exist.
|
||||
Old token is located at ["api"]["token"] in userdata.json.
|
||||
tokens.json path is declared in TOKENS_FILE imported from utils.py
|
||||
tokens.json must have the following format:
|
||||
{
|
||||
"tokens": [
|
||||
{
|
||||
"token": "token_string",
|
||||
"name": "Master Token",
|
||||
"date": "current date from str(datetime.now())",
|
||||
}
|
||||
]
|
||||
}
|
||||
tokens.json must have 0600 permissions.
|
||||
"""
|
||||
|
||||
def is_migration_needed(self):
|
||||
return not os.path.exists(TOKENS_FILE)
|
||||
|
||||
def migrate(self):
|
||||
try:
|
||||
print(f"Creating tokens.json file at {TOKENS_FILE}")
|
||||
with ReadUserData() as userdata:
|
||||
token = userdata["api"]["token"]
|
||||
# Touch tokens.json with 0600 permissions
|
||||
Path(TOKENS_FILE).touch(mode=0o600)
|
||||
# Write token to tokens.json
|
||||
structure = {
|
||||
"tokens": [
|
||||
{
|
||||
"token": token,
|
||||
"name": "primary_token",
|
||||
"date": str(datetime.now()),
|
||||
}
|
||||
]
|
||||
}
|
||||
with open(TOKENS_FILE, "w", encoding="utf-8") as tokens:
|
||||
json.dump(structure, tokens, indent=4)
|
||||
print("Done")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("Error creating tokens.json")
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
|
||||
|
||||
class FixNixosConfigBranch(Migration):
|
||||
def get_migration_name(self):
|
||||
return "fix_nixos_config_branch"
|
||||
|
||||
def get_migration_description(self):
|
||||
return """Mobile SelfPrivacy app introduced a bug in version 0.4.0.
|
||||
New servers were initialized with a rolling-testing nixos config branch.
|
||||
This was fixed in app version 0.4.2, but existing servers were not updated.
|
||||
This migration fixes this by changing the nixos config branch to master.
|
||||
"""
|
||||
|
||||
def is_migration_needed(self):
|
||||
"""Check the current branch of /etc/nixos and return True if it is rolling-testing"""
|
||||
current_working_directory = os.getcwd()
|
||||
try:
|
||||
os.chdir("/etc/nixos")
|
||||
nixos_config_branch = subprocess.check_output(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True
|
||||
)
|
||||
os.chdir(current_working_directory)
|
||||
return nixos_config_branch.decode("utf-8").strip() == "rolling-testing"
|
||||
except subprocess.CalledProcessError:
|
||||
os.chdir(current_working_directory)
|
||||
return False
|
||||
|
||||
def migrate(self):
|
||||
"""Affected server pulled the config with the --single-branch flag.
|
||||
Git config remote.origin.fetch has to be changed, so all branches will be fetched.
|
||||
Then, fetch all branches, pull and switch to master branch.
|
||||
"""
|
||||
print("Fixing Nixos config branch")
|
||||
current_working_directory = os.getcwd()
|
||||
try:
|
||||
os.chdir("/etc/nixos")
|
||||
|
||||
subprocess.check_output(
|
||||
[
|
||||
"git",
|
||||
"config",
|
||||
"remote.origin.fetch",
|
||||
"+refs/heads/*:refs/remotes/origin/*",
|
||||
]
|
||||
)
|
||||
subprocess.check_output(["git", "fetch", "--all"])
|
||||
subprocess.check_output(["git", "pull"])
|
||||
subprocess.check_output(["git", "checkout", "master"])
|
||||
os.chdir(current_working_directory)
|
||||
print("Done")
|
||||
except subprocess.CalledProcessError:
|
||||
os.chdir(current_working_directory)
|
||||
print("Error")
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
|
||||
|
||||
class MigrateToSelfprivacyChannel(Migration):
|
||||
"""Migrate to selfprivacy Nix channel."""
|
||||
|
||||
def get_migration_name(self):
|
||||
return "migrate_to_selfprivacy_channel"
|
||||
|
||||
def get_migration_description(self):
|
||||
return "Migrate to selfprivacy Nix channel."
|
||||
|
||||
def is_migration_needed(self):
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
["nix-channel", "--list"], start_new_session=True
|
||||
)
|
||||
output = output.decode("utf-8")
|
||||
first_line = output.split("\n", maxsplit=1)[0]
|
||||
return first_line.startswith("nixos") and (
|
||||
first_line.endswith("nixos-21.11") or first_line.endswith("nixos-21.05")
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
def migrate(self):
|
||||
# Change the channel and update them.
|
||||
# Also, go to /etc/nixos directory and make a git pull
|
||||
current_working_directory = os.getcwd()
|
||||
try:
|
||||
print("Changing channel")
|
||||
os.chdir("/etc/nixos")
|
||||
subprocess.check_output(
|
||||
[
|
||||
"nix-channel",
|
||||
"--add",
|
||||
"https://channel.selfprivacy.org/nixos-selfpricacy",
|
||||
"nixos",
|
||||
]
|
||||
)
|
||||
subprocess.check_output(["nix-channel", "--update"])
|
||||
subprocess.check_output(["git", "pull"])
|
||||
os.chdir(current_working_directory)
|
||||
except subprocess.CalledProcessError:
|
||||
os.chdir(current_working_directory)
|
||||
print("Error")
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
from selfprivacy_api.utils import ReadUserData, WriteUserData
|
||||
from selfprivacy_api.utils.block_devices import BlockDevices
|
||||
|
||||
|
||||
class MountVolume(Migration):
|
||||
"""Mount volume."""
|
||||
|
||||
def get_migration_name(self):
|
||||
return "mount_volume"
|
||||
|
||||
def get_migration_description(self):
|
||||
return "Mount volume if it is not mounted."
|
||||
|
||||
def is_migration_needed(self):
|
||||
try:
|
||||
with ReadUserData() as userdata:
|
||||
return "volumes" not in userdata
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
def migrate(self):
|
||||
# Get info about existing volumes
|
||||
# Write info about volumes to userdata.json
|
||||
try:
|
||||
volumes = BlockDevices().get_block_devices()
|
||||
# If there is an unmounted volume sdb,
|
||||
# Write it to userdata.json
|
||||
is_there_a_volume = False
|
||||
for volume in volumes:
|
||||
if volume.name == "sdb":
|
||||
is_there_a_volume = True
|
||||
break
|
||||
with WriteUserData() as userdata:
|
||||
userdata["volumes"] = []
|
||||
if is_there_a_volume:
|
||||
userdata["volumes"].append(
|
||||
{
|
||||
"device": "/dev/sdb",
|
||||
"mountPoint": "/volumes/sdb",
|
||||
"fsType": "ext4",
|
||||
}
|
||||
)
|
||||
print("Done")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("Error mounting volume")
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from selfprivacy_api.migrations.migration import Migration
|
||||
from selfprivacy_api.models.tokens.token import Token
|
||||
|
||||
from selfprivacy_api.repositories.tokens.redis_tokens_repository import (
|
||||
RedisTokensRepository,
|
||||
)
|
||||
from selfprivacy_api.repositories.tokens.abstract_tokens_repository import (
|
||||
AbstractTokensRepository,
|
||||
)
|
||||
from selfprivacy_api.utils import ReadUserData, UserDataFiles
|
||||
|
||||
|
||||
class WriteTokenToRedis(Migration):
|
||||
"""Load Json tokens into Redis"""
|
||||
|
||||
def get_migration_name(self):
|
||||
return "write_token_to_redis"
|
||||
|
||||
def get_migration_description(self):
|
||||
return "Loads the initial token into redis token storage"
|
||||
|
||||
def is_repo_empty(self, repo: AbstractTokensRepository) -> bool:
|
||||
if repo.get_tokens() != []:
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_token_from_json(self) -> Optional[Token]:
|
||||
try:
|
||||
with ReadUserData(UserDataFiles.SECRETS) as userdata:
|
||||
return Token(
|
||||
token=userdata["api"]["token"],
|
||||
device_name="Initial device",
|
||||
created_at=datetime.now(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return None
|
||||
|
||||
def is_migration_needed(self):
|
||||
try:
|
||||
if self.get_token_from_json() is not None and self.is_repo_empty(
|
||||
RedisTokensRepository()
|
||||
):
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
def migrate(self):
|
||||
# Write info about providers to userdata.json
|
||||
try:
|
||||
token = self.get_token_from_json()
|
||||
if token is None:
|
||||
print("No token found in secrets.json")
|
||||
return
|
||||
RedisTokensRepository()._store_token(token)
|
||||
|
||||
print("Done")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("Error migrating access tokens from json to redis")
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
"""for storage in Redis"""
|
||||
|
||||
|
||||
class BackupProviderModel(BaseModel):
|
||||
kind: str
|
||||
login: str
|
||||
key: str
|
||||
location: str
|
||||
repo_id: str # for app usage, not for us
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import datetime
|
||||
from pydantic import BaseModel
|
||||
|
||||
from selfprivacy_api.graphql.common_types.backup import BackupReason
|
||||
|
||||
|
||||
class Snapshot(BaseModel):
|
||||
id: str
|
||||
service_name: str
|
||||
created_at: datetime.datetime
|
||||
reason: BackupReason = BackupReason.EXPLICIT
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
from enum import Enum
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ServiceStatus(Enum):
|
||||
"""Enum for service status"""
|
||||
|
||||
ACTIVE = "ACTIVE"
|
||||
RELOADING = "RELOADING"
|
||||
INACTIVE = "INACTIVE"
|
||||
FAILED = "FAILED"
|
||||
ACTIVATING = "ACTIVATING"
|
||||
DEACTIVATING = "DEACTIVATING"
|
||||
OFF = "OFF"
|
||||
|
||||
|
||||
class ServiceDnsRecord(BaseModel):
|
||||
type: str
|
||||
name: str
|
||||
content: str
|
||||
ttl: int
|
||||
display_name: str
|
||||
priority: Optional[int] = None
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
"""
|
||||
New device key used to obtain access token.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import secrets
|
||||
from pydantic import BaseModel
|
||||
from mnemonic import Mnemonic
|
||||
|
||||
from selfprivacy_api.models.tokens.time import is_past
|
||||
|
||||
|
||||
class NewDeviceKey(BaseModel):
|
||||
"""
|
||||
Recovery key used to obtain access token.
|
||||
|
||||
Recovery key has a key string, date of creation, date of expiration.
|
||||
"""
|
||||
|
||||
key: str
|
||||
created_at: datetime
|
||||
expires_at: datetime
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
Check if key is valid.
|
||||
"""
|
||||
if is_past(self.expires_at):
|
||||
return False
|
||||
return True
|
||||
|
||||
def as_mnemonic(self) -> str:
|
||||
"""
|
||||
Get the key as a mnemonic.
|
||||
"""
|
||||
return Mnemonic(language="english").to_mnemonic(bytes.fromhex(self.key))
|
||||
|
||||
@staticmethod
|
||||
def generate() -> "NewDeviceKey":
|
||||
"""
|
||||
Factory to generate a random token.
|
||||
"""
|
||||
creation_date = datetime.now(timezone.utc)
|
||||
key = secrets.token_bytes(16).hex()
|
||||
return NewDeviceKey(
|
||||
key=key,
|
||||
created_at=creation_date,
|
||||
expires_at=creation_date + timedelta(minutes=10),
|
||||
)
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
"""
|
||||
Recovery key used to obtain access token.
|
||||
|
||||
Recovery key has a token string, date of creation, optional date of expiration and optional count of uses left.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
import secrets
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
from mnemonic import Mnemonic
|
||||
|
||||
from selfprivacy_api.models.tokens.time import is_past, ensure_timezone
|
||||
|
||||
|
||||
class RecoveryKey(BaseModel):
|
||||
"""
|
||||
Recovery key used to obtain access token.
|
||||
|
||||
Recovery key has a key string, date of creation, optional date of expiration and optional count of uses left.
|
||||
"""
|
||||
|
||||
key: str
|
||||
created_at: datetime
|
||||
expires_at: Optional[datetime]
|
||||
uses_left: Optional[int]
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
Check if the recovery key is valid.
|
||||
"""
|
||||
if self.expires_at is not None and is_past(self.expires_at):
|
||||
return False
|
||||
if self.uses_left is not None and self.uses_left <= 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
def as_mnemonic(self) -> str:
|
||||
"""
|
||||
Get the recovery key as a mnemonic.
|
||||
"""
|
||||
return Mnemonic(language="english").to_mnemonic(bytes.fromhex(self.key))
|
||||
|
||||
@staticmethod
|
||||
def generate(
|
||||
expiration: Optional[datetime],
|
||||
uses_left: Optional[int],
|
||||
) -> "RecoveryKey":
|
||||
"""
|
||||
Factory to generate a random token.
|
||||
If passed naive time as expiration, assumes utc
|
||||
"""
|
||||
creation_date = datetime.now(timezone.utc)
|
||||
if expiration is not None:
|
||||
expiration = ensure_timezone(expiration)
|
||||
key = secrets.token_bytes(24).hex()
|
||||
return RecoveryKey(
|
||||
key=key,
|
||||
created_at=creation_date,
|
||||
expires_at=expiration,
|
||||
uses_left=uses_left,
|
||||
)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def is_past(dt: datetime) -> bool:
|
||||
# we cannot compare a naive now()
|
||||
# to dt which might be tz-aware or unaware
|
||||
dt = ensure_timezone(dt)
|
||||
return dt < datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_timezone(dt: datetime) -> datetime:
|
||||
if dt.tzinfo is None or dt.tzinfo.utcoffset(None) is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
"""
|
||||
Model of the access token.
|
||||
|
||||
Access token has a token string, device name and date of creation.
|
||||
"""
|
||||
from datetime import datetime
|
||||
import secrets
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Token(BaseModel):
|
||||
"""
|
||||
Model of the access token.
|
||||
|
||||
Access token has a token string, device name and date of creation.
|
||||
"""
|
||||
|
||||
token: str
|
||||
device_name: str
|
||||
created_at: datetime
|
||||
|
||||
@staticmethod
|
||||
def generate(device_name: str) -> "Token":
|
||||
"""
|
||||
Factory to generate a random token.
|
||||
"""
|
||||
creation_date = datetime.now()
|
||||
token = secrets.token_urlsafe(32)
|
||||
return Token(
|
||||
token=token,
|
||||
device_name=device_name,
|
||||
created_at=creation_date,
|
||||
)
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from mnemonic import Mnemonic
|
||||
from secrets import randbelow
|
||||
import re
|
||||
|
||||
from selfprivacy_api.models.tokens.token import Token
|
||||
from selfprivacy_api.repositories.tokens.exceptions import (
|
||||
TokenNotFound,
|
||||
InvalidMnemonic,
|
||||
RecoveryKeyNotFound,
|
||||
NewDeviceKeyNotFound,
|
||||
)
|
||||
from selfprivacy_api.models.tokens.recovery_key import RecoveryKey
|
||||
from selfprivacy_api.models.tokens.new_device_key import NewDeviceKey
|
||||
|
||||
|
||||
class AbstractTokensRepository(ABC):
|
||||
def get_token_by_token_string(self, token_string: str) -> Token:
|
||||
"""Get the token by token"""
|
||||
tokens = self.get_tokens()
|
||||
for token in tokens:
|
||||
if token.token == token_string:
|
||||
return token
|
||||
|
||||
raise TokenNotFound("Token not found!")
|
||||
|
||||
def get_token_by_name(self, token_name: str) -> Token:
|
||||
"""Get the token by name"""
|
||||
tokens = self.get_tokens()
|
||||
for token in tokens:
|
||||
if token.device_name == token_name:
|
||||
return token
|
||||
|
||||
raise TokenNotFound("Token not found!")
|
||||
|
||||
@abstractmethod
|
||||
def get_tokens(self) -> list[Token]:
|
||||
"""Get the tokens"""
|
||||
|
||||
def create_token(self, device_name: str) -> Token:
|
||||
"""Create new token"""
|
||||
unique_name = self._make_unique_device_name(device_name)
|
||||
new_token = Token.generate(unique_name)
|
||||
|
||||
self._store_token(new_token)
|
||||
|
||||
return new_token
|
||||
|
||||
@abstractmethod
|
||||
def delete_token(self, input_token: Token) -> None:
|
||||
"""Delete the token"""
|
||||
|
||||
def refresh_token(self, input_token: Token) -> Token:
|
||||
"""Change the token field of the existing token"""
|
||||
new_token = Token.generate(device_name=input_token.device_name)
|
||||
new_token.created_at = input_token.created_at
|
||||
|
||||
if input_token in self.get_tokens():
|
||||
self.delete_token(input_token)
|
||||
self._store_token(new_token)
|
||||
return new_token
|
||||
|
||||
raise TokenNotFound("Token not found!")
|
||||
|
||||
def is_token_valid(self, token_string: str) -> bool:
|
||||
"""Check if the token is valid"""
|
||||
return token_string in [token.token for token in self.get_tokens()]
|
||||
|
||||
def is_token_name_exists(self, token_name: str) -> bool:
|
||||
"""Check if the token name exists"""
|
||||
return token_name in [token.device_name for token in self.get_tokens()]
|
||||
|
||||
def is_token_name_pair_valid(self, token_name: str, token_string: str) -> bool:
|
||||
"""Check if the token name and token are valid"""
|
||||
try:
|
||||
token = self.get_token_by_name(token_name)
|
||||
if token is None:
|
||||
return False
|
||||
except TokenNotFound:
|
||||
return False
|
||||
return token.token == token_string
|
||||
|
||||
@abstractmethod
|
||||
def get_recovery_key(self) -> Optional[RecoveryKey]:
|
||||
"""Get the recovery key"""
|
||||
|
||||
def create_recovery_key(
|
||||
self,
|
||||
expiration: Optional[datetime],
|
||||
uses_left: Optional[int],
|
||||
) -> RecoveryKey:
|
||||
"""Create the recovery key"""
|
||||
recovery_key = RecoveryKey.generate(expiration, uses_left)
|
||||
self._store_recovery_key(recovery_key)
|
||||
return recovery_key
|
||||
|
||||
def use_mnemonic_recovery_key(
|
||||
self, mnemonic_phrase: str, device_name: str
|
||||
) -> Token:
|
||||
"""Use the mnemonic recovery key and create a new token with the given name"""
|
||||
if not self.is_recovery_key_valid():
|
||||
raise RecoveryKeyNotFound("Recovery key not found")
|
||||
|
||||
recovery_key = self.get_recovery_key()
|
||||
|
||||
if recovery_key is None:
|
||||
raise RecoveryKeyNotFound("Recovery key not found")
|
||||
|
||||
recovery_hex_key = recovery_key.key
|
||||
if not self._assert_mnemonic(recovery_hex_key, mnemonic_phrase):
|
||||
raise RecoveryKeyNotFound("Recovery key not found")
|
||||
|
||||
new_token = self.create_token(device_name=device_name)
|
||||
|
||||
self._decrement_recovery_token()
|
||||
|
||||
return new_token
|
||||
|
||||
def is_recovery_key_valid(self) -> bool:
|
||||
"""Check if the recovery key is valid"""
|
||||
recovery_key = self.get_recovery_key()
|
||||
if recovery_key is None:
|
||||
return False
|
||||
return recovery_key.is_valid()
|
||||
|
||||
@abstractmethod
|
||||
def _store_recovery_key(self, recovery_key: RecoveryKey) -> None:
|
||||
"""Store recovery key directly"""
|
||||
|
||||
@abstractmethod
|
||||
def _delete_recovery_key(self) -> None:
|
||||
"""Delete the recovery key"""
|
||||
|
||||
def get_new_device_key(self) -> NewDeviceKey:
|
||||
"""Creates and returns the new device key"""
|
||||
new_device_key = NewDeviceKey.generate()
|
||||
self._store_new_device_key(new_device_key)
|
||||
|
||||
return new_device_key
|
||||
|
||||
def _store_new_device_key(self, new_device_key: NewDeviceKey) -> None:
|
||||
"""Store new device key directly"""
|
||||
|
||||
@abstractmethod
|
||||
def delete_new_device_key(self) -> None:
|
||||
"""Delete the new device key"""
|
||||
|
||||
def use_mnemonic_new_device_key(
|
||||
self, mnemonic_phrase: str, device_name: str
|
||||
) -> Token:
|
||||
"""Use the mnemonic new device key"""
|
||||
new_device_key = self._get_stored_new_device_key()
|
||||
if not new_device_key:
|
||||
raise NewDeviceKeyNotFound
|
||||
|
||||
if not new_device_key.is_valid():
|
||||
raise NewDeviceKeyNotFound
|
||||
|
||||
if not self._assert_mnemonic(new_device_key.key, mnemonic_phrase):
|
||||
raise NewDeviceKeyNotFound("Phrase is not token!")
|
||||
|
||||
new_token = self.create_token(device_name=device_name)
|
||||
self.delete_new_device_key()
|
||||
|
||||
return new_token
|
||||
|
||||
def reset(self):
|
||||
for token in self.get_tokens():
|
||||
self.delete_token(token)
|
||||
self.delete_new_device_key()
|
||||
self._delete_recovery_key()
|
||||
|
||||
def clone(self, source: AbstractTokensRepository) -> None:
|
||||
"""Clone the state of another repository to this one"""
|
||||
self.reset()
|
||||
for token in source.get_tokens():
|
||||
self._store_token(token)
|
||||
|
||||
recovery_key = source.get_recovery_key()
|
||||
if recovery_key is not None:
|
||||
self._store_recovery_key(recovery_key)
|
||||
|
||||
new_device_key = source._get_stored_new_device_key()
|
||||
if new_device_key is not None:
|
||||
self._store_new_device_key(new_device_key)
|
||||
|
||||
@abstractmethod
|
||||
def _store_token(self, new_token: Token):
|
||||
"""Store a token directly"""
|
||||
|
||||
@abstractmethod
|
||||
def _decrement_recovery_token(self):
|
||||
"""Decrement recovery key use count by one"""
|
||||
|
||||
@abstractmethod
|
||||
def _get_stored_new_device_key(self) -> Optional[NewDeviceKey]:
|
||||
"""Retrieves new device key that is already stored."""
|
||||
|
||||
def _make_unique_device_name(self, name: str) -> str:
|
||||
"""Token name must be an alphanumeric string and not empty.
|
||||
Replace invalid characters with '_'
|
||||
If name exists, add a random number to the end of the name until it is unique.
|
||||
"""
|
||||
if not re.match("^[a-zA-Z0-9]*$", name):
|
||||
name = re.sub("[^a-zA-Z0-9]", "_", name)
|
||||
if name == "":
|
||||
name = "Unknown device"
|
||||
while self.is_token_name_exists(name):
|
||||
name += str(randbelow(10))
|
||||
return name
|
||||
|
||||
# TODO: find a proper place for it
|
||||
def _assert_mnemonic(self, hex_key: str, mnemonic_phrase: str):
|
||||
"""Return true if hex string matches the phrase, false otherwise
|
||||
Raise an InvalidMnemonic error if not mnemonic"""
|
||||
recovery_token = bytes.fromhex(hex_key)
|
||||
if not Mnemonic(language="english").check(mnemonic_phrase):
|
||||
raise InvalidMnemonic("Phrase is not mnemonic!")
|
||||
|
||||
phrase_bytes = Mnemonic(language="english").to_entropy(mnemonic_phrase)
|
||||
return phrase_bytes == recovery_token
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
class TokenNotFound(Exception):
|
||||
"""Token not found!"""
|
||||
|
||||
|
||||
class RecoveryKeyNotFound(Exception):
|
||||
"""Recovery key not found!"""
|
||||
|
||||
|
||||
class InvalidMnemonic(Exception):
|
||||
"""Phrase is not mnemonic!"""
|
||||
|
||||
|
||||
class NewDeviceKeyNotFound(Exception):
|
||||
"""New device key not found!"""
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
"""
|
||||
Token repository using Redis as backend.
|
||||
"""
|
||||
from typing import Any, Optional
|
||||
from datetime import datetime
|
||||
from hashlib import md5
|
||||
from datetime import timezone
|
||||
|
||||
from selfprivacy_api.repositories.tokens.abstract_tokens_repository import (
|
||||
AbstractTokensRepository,
|
||||
)
|
||||
from selfprivacy_api.utils.redis_pool import RedisPool
|
||||
from selfprivacy_api.models.tokens.token import Token
|
||||
from selfprivacy_api.models.tokens.recovery_key import RecoveryKey
|
||||
from selfprivacy_api.models.tokens.new_device_key import NewDeviceKey
|
||||
from selfprivacy_api.repositories.tokens.exceptions import TokenNotFound
|
||||
|
||||
TOKENS_PREFIX = "token_repo:tokens:"
|
||||
NEW_DEVICE_KEY_REDIS_KEY = "token_repo:new_device_key"
|
||||
RECOVERY_KEY_REDIS_KEY = "token_repo:recovery_key"
|
||||
|
||||
|
||||
class RedisTokensRepository(AbstractTokensRepository):
|
||||
"""
|
||||
Token repository using Redis as a backend
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.connection = RedisPool().get_connection()
|
||||
|
||||
@staticmethod
|
||||
def token_key_for_device(device_name: str):
|
||||
md5_hash = md5(usedforsecurity=False)
|
||||
md5_hash.update(bytes(device_name, "utf-8"))
|
||||
digest = md5_hash.hexdigest()
|
||||
return TOKENS_PREFIX + digest
|
||||
|
||||
def get_tokens(self) -> list[Token]:
|
||||
"""Get the tokens"""
|
||||
redis = self.connection
|
||||
token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*") # type: ignore
|
||||
tokens = []
|
||||
for key in token_keys:
|
||||
token = self._token_from_hash(key)
|
||||
if token is not None:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
def _discover_token_key(self, input_token: Token) -> Optional[str]:
|
||||
"""brute-force searching for tokens, for robust deletion"""
|
||||
redis = self.connection
|
||||
token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*") # type: ignore
|
||||
for key in token_keys:
|
||||
token = self._token_from_hash(key)
|
||||
if token == input_token:
|
||||
return key
|
||||
return None
|
||||
|
||||
def delete_token(self, input_token: Token) -> None:
|
||||
"""Delete the token"""
|
||||
redis = self.connection
|
||||
key = self._discover_token_key(input_token)
|
||||
if key is None:
|
||||
raise TokenNotFound
|
||||
redis.delete(key)
|
||||
|
||||
def get_recovery_key(self) -> Optional[RecoveryKey]:
|
||||
"""Get the recovery key"""
|
||||
redis = self.connection
|
||||
if redis.exists(RECOVERY_KEY_REDIS_KEY):
|
||||
return self._recovery_key_from_hash(RECOVERY_KEY_REDIS_KEY)
|
||||
return None
|
||||
|
||||
def _store_recovery_key(self, recovery_key: RecoveryKey) -> None:
|
||||
self._store_model_as_hash(RECOVERY_KEY_REDIS_KEY, recovery_key)
|
||||
|
||||
def _delete_recovery_key(self) -> None:
|
||||
"""Delete the recovery key"""
|
||||
redis = self.connection
|
||||
redis.delete(RECOVERY_KEY_REDIS_KEY)
|
||||
|
||||
def _store_new_device_key(self, new_device_key: NewDeviceKey) -> None:
|
||||
"""Store new device key directly"""
|
||||
self._store_model_as_hash(NEW_DEVICE_KEY_REDIS_KEY, new_device_key)
|
||||
|
||||
def delete_new_device_key(self) -> None:
|
||||
"""Delete the new device key"""
|
||||
redis = self.connection
|
||||
redis.delete(NEW_DEVICE_KEY_REDIS_KEY)
|
||||
|
||||
@staticmethod
|
||||
def _token_redis_key(token: Token) -> str:
|
||||
return RedisTokensRepository.token_key_for_device(token.device_name)
|
||||
|
||||
def _store_token(self, new_token: Token):
|
||||
"""Store a token directly"""
|
||||
key = RedisTokensRepository._token_redis_key(new_token)
|
||||
self._store_model_as_hash(key, new_token)
|
||||
|
||||
def _decrement_recovery_token(self):
|
||||
"""Decrement recovery key use count by one"""
|
||||
if self.is_recovery_key_valid():
|
||||
recovery_key = self.get_recovery_key()
|
||||
if recovery_key is None:
|
||||
return
|
||||
uses_left = recovery_key.uses_left
|
||||
if uses_left is not None:
|
||||
redis = self.connection
|
||||
redis.hset(RECOVERY_KEY_REDIS_KEY, "uses_left", uses_left - 1)
|
||||
|
||||
def _get_stored_new_device_key(self) -> Optional[NewDeviceKey]:
|
||||
"""Retrieves new device key that is already stored."""
|
||||
return self._new_device_key_from_hash(NEW_DEVICE_KEY_REDIS_KEY)
|
||||
|
||||
@staticmethod
|
||||
def _is_date_key(key: str) -> bool:
|
||||
return key in [
|
||||
"created_at",
|
||||
"expires_at",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _prepare_model_dict(model_dict: dict[str, Any]) -> None:
|
||||
date_keys = [
|
||||
key for key in model_dict.keys() if RedisTokensRepository._is_date_key(key)
|
||||
]
|
||||
for date in date_keys:
|
||||
if model_dict[date] != "None":
|
||||
model_dict[date] = datetime.fromisoformat(model_dict[date])
|
||||
for key in model_dict.keys():
|
||||
if model_dict[key] == "None":
|
||||
model_dict[key] = None
|
||||
|
||||
def _model_dict_from_hash(self, redis_key: str) -> Optional[dict[str, Any]]:
|
||||
redis = self.connection
|
||||
if redis.exists(redis_key):
|
||||
token_dict: dict[str, Any] = redis.hgetall(redis_key) # type: ignore
|
||||
RedisTokensRepository._prepare_model_dict(token_dict)
|
||||
return token_dict
|
||||
return None
|
||||
|
||||
def _hash_as_model(self, redis_key: str, model_class):
|
||||
token_dict = self._model_dict_from_hash(redis_key)
|
||||
if token_dict is not None:
|
||||
return model_class(**token_dict)
|
||||
return None
|
||||
|
||||
def _token_from_hash(self, redis_key: str) -> Optional[Token]:
|
||||
token = self._hash_as_model(redis_key, Token)
|
||||
if token is not None:
|
||||
token.created_at = token.created_at.replace(tzinfo=None)
|
||||
return token
|
||||
return None
|
||||
|
||||
def _recovery_key_from_hash(self, redis_key: str) -> Optional[RecoveryKey]:
|
||||
return self._hash_as_model(redis_key, RecoveryKey)
|
||||
|
||||
def _new_device_key_from_hash(self, redis_key: str) -> Optional[NewDeviceKey]:
|
||||
return self._hash_as_model(redis_key, NewDeviceKey)
|
||||
|
||||
def _store_model_as_hash(self, redis_key, model):
|
||||
redis = self.connection
|
||||
for key, value in model.dict().items():
|
||||
if isinstance(value, datetime):
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
value = value.isoformat()
|
||||
redis.hset(redis_key, key, str(value))
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from selfprivacy_api.actions.api_tokens import (
|
||||
CannotDeleteCallerException,
|
||||
InvalidExpirationDate,
|
||||
InvalidUsesLeft,
|
||||
NotFoundException,
|
||||
delete_api_token,
|
||||
get_api_recovery_token_status,
|
||||
get_api_tokens_with_caller_flag,
|
||||
get_new_api_recovery_key,
|
||||
refresh_api_token,
|
||||
)
|
||||
|
||||
from selfprivacy_api.dependencies import TokenHeader, get_token_header
|
||||
|
||||
from selfprivacy_api.utils.auth import (
|
||||
delete_new_device_auth_token,
|
||||
get_new_device_auth_token,
|
||||
use_mnemonic_recoverery_token,
|
||||
use_new_device_auth_token,
|
||||
)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/auth",
|
||||
tags=["auth"],
|
||||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tokens")
|
||||
async def rest_get_tokens(auth_token: TokenHeader = Depends(get_token_header)):
|
||||
"""Get the tokens info"""
|
||||
return get_api_tokens_with_caller_flag(auth_token.token)
|
||||
|
||||
|
||||
class DeleteTokenInput(BaseModel):
|
||||
"""Delete token input"""
|
||||
|
||||
token_name: str
|
||||
|
||||
|
||||
@router.delete("/tokens")
|
||||
async def rest_delete_tokens(
|
||||
token: DeleteTokenInput, auth_token: TokenHeader = Depends(get_token_header)
|
||||
):
|
||||
"""Delete the tokens"""
|
||||
try:
|
||||
delete_api_token(auth_token.token, token.token_name)
|
||||
except NotFoundException:
|
||||
raise HTTPException(status_code=404, detail="Token not found")
|
||||
except CannotDeleteCallerException:
|
||||
raise HTTPException(status_code=400, detail="Cannot delete caller's token")
|
||||
return {"message": "Token deleted"}
|
||||
|
||||
|
||||
@router.post("/tokens")
|
||||
async def rest_refresh_token(auth_token: TokenHeader = Depends(get_token_header)):
|
||||
"""Refresh the token"""
|
||||
try:
|
||||
new_token = refresh_api_token(auth_token.token)
|
||||
except NotFoundException:
|
||||
raise HTTPException(status_code=404, detail="Token not found")
|
||||
return {"token": new_token}
|
||||
|
||||
|
||||
@router.get("/recovery_token")
|
||||
async def rest_get_recovery_token_status(
|
||||
auth_token: TokenHeader = Depends(get_token_header),
|
||||
):
|
||||
return get_api_recovery_token_status()
|
||||
|
||||
|
||||
class CreateRecoveryTokenInput(BaseModel):
|
||||
expiration: Optional[datetime] = None
|
||||
uses: Optional[int] = None
|
||||
|
||||
|
||||
@router.post("/recovery_token")
|
||||
async def rest_create_recovery_token(
|
||||
limits: CreateRecoveryTokenInput = CreateRecoveryTokenInput(),
|
||||
auth_token: TokenHeader = Depends(get_token_header),
|
||||
):
|
||||
try:
|
||||
token = get_new_api_recovery_key(limits.expiration, limits.uses)
|
||||
except InvalidExpirationDate as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except InvalidUsesLeft as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return {"token": token}
|
||||
|
||||
|
||||
class UseTokenInput(BaseModel):
|
||||
token: str
|
||||
device: str
|
||||
|
||||
|
||||
@router.post("/recovery_token/use")
|
||||
async def rest_use_recovery_token(input: UseTokenInput):
|
||||
token = use_mnemonic_recoverery_token(input.token, input.device)
|
||||
if token is None:
|
||||
raise HTTPException(status_code=404, detail="Token not found")
|
||||
return {"token": token}
|
||||
|
||||
|
||||
@router.post("/new_device")
|
||||
async def rest_new_device(auth_token: TokenHeader = Depends(get_token_header)):
|
||||
token = get_new_device_auth_token()
|
||||
return {"token": token}
|
||||
|
||||
|
||||
@router.delete("/new_device")
|
||||
async def rest_delete_new_device_token(
|
||||
auth_token: TokenHeader = Depends(get_token_header),
|
||||
):
|
||||
delete_new_device_auth_token()
|
||||
return {"token": None}
|
||||
|
||||
|
||||
@router.post("/new_device/authorize")
|
||||
async def rest_new_device_authorize(input: UseTokenInput):
|
||||
token = use_new_device_auth_token(input.token, input.device)
|
||||
if token is None:
|
||||
raise HTTPException(status_code=404, detail="Token not found")
|
||||
return {"message": "Device authorized", "token": token}
|
||||
|
|
@ -0,0 +1,373 @@
|
|||
"""Basic services legacy api"""
|
||||
import base64
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from selfprivacy_api.actions.ssh import (
|
||||
InvalidPublicKey,
|
||||
KeyAlreadyExists,
|
||||
KeyNotFound,
|
||||
create_ssh_key,
|
||||
enable_ssh,
|
||||
get_ssh_settings,
|
||||
remove_ssh_key,
|
||||
set_ssh_settings,
|
||||
)
|
||||
from selfprivacy_api.actions.users import UserNotFound, get_user_by_username
|
||||
|
||||
from selfprivacy_api.dependencies import get_token_header
|
||||
from selfprivacy_api.restic_controller import ResticController, ResticStates
|
||||
from selfprivacy_api.restic_controller import tasks as restic_tasks
|
||||
from selfprivacy_api.services.bitwarden import Bitwarden
|
||||
from selfprivacy_api.services.gitea import Gitea
|
||||
from selfprivacy_api.services.mailserver import MailServer
|
||||
from selfprivacy_api.services.nextcloud import Nextcloud
|
||||
from selfprivacy_api.services.ocserv import Ocserv
|
||||
from selfprivacy_api.services.pleroma import Pleroma
|
||||
from selfprivacy_api.services.service import ServiceStatus
|
||||
from selfprivacy_api.utils import WriteUserData, get_dkim_key, get_domain
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/services",
|
||||
tags=["services"],
|
||||
dependencies=[Depends(get_token_header)],
|
||||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
|
||||
def service_status_to_return_code(status: ServiceStatus):
|
||||
"""Converts service status object to return code for
|
||||
compatibility with legacy api"""
|
||||
if status == ServiceStatus.ACTIVE:
|
||||
return 0
|
||||
elif status == ServiceStatus.FAILED:
|
||||
return 1
|
||||
elif status == ServiceStatus.INACTIVE:
|
||||
return 3
|
||||
elif status == ServiceStatus.OFF:
|
||||
return 4
|
||||
else:
|
||||
return 2
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def get_status():
|
||||
"""Get the status of the services"""
|
||||
mail_status = MailServer.get_status()
|
||||
bitwarden_status = Bitwarden.get_status()
|
||||
gitea_status = Gitea.get_status()
|
||||
nextcloud_status = Nextcloud.get_status()
|
||||
ocserv_stauts = Ocserv.get_status()
|
||||
pleroma_status = Pleroma.get_status()
|
||||
|
||||
return {
|
||||
"imap": service_status_to_return_code(mail_status),
|
||||
"smtp": service_status_to_return_code(mail_status),
|
||||
"http": 0,
|
||||
"bitwarden": service_status_to_return_code(bitwarden_status),
|
||||
"gitea": service_status_to_return_code(gitea_status),
|
||||
"nextcloud": service_status_to_return_code(nextcloud_status),
|
||||
"ocserv": service_status_to_return_code(ocserv_stauts),
|
||||
"pleroma": service_status_to_return_code(pleroma_status),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/bitwarden/enable")
|
||||
async def enable_bitwarden():
|
||||
"""Enable Bitwarden"""
|
||||
Bitwarden.enable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Bitwarden enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/bitwarden/disable")
|
||||
async def disable_bitwarden():
|
||||
"""Disable Bitwarden"""
|
||||
Bitwarden.disable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Bitwarden disabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/gitea/enable")
|
||||
async def enable_gitea():
|
||||
"""Enable Gitea"""
|
||||
Gitea.enable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Gitea enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/gitea/disable")
|
||||
async def disable_gitea():
|
||||
"""Disable Gitea"""
|
||||
Gitea.disable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Gitea disabled",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/mailserver/dkim")
|
||||
async def get_mailserver_dkim():
|
||||
"""Get the DKIM record for the mailserver"""
|
||||
domain = get_domain()
|
||||
|
||||
dkim = get_dkim_key(domain)
|
||||
if dkim is None:
|
||||
raise HTTPException(status_code=404, detail="DKIM record not found")
|
||||
dkim = base64.b64encode(dkim.encode("utf-8")).decode("utf-8")
|
||||
return dkim
|
||||
|
||||
|
||||
@router.post("/nextcloud/enable")
|
||||
async def enable_nextcloud():
|
||||
"""Enable Nextcloud"""
|
||||
Nextcloud.enable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Nextcloud enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/nextcloud/disable")
|
||||
async def disable_nextcloud():
|
||||
"""Disable Nextcloud"""
|
||||
Nextcloud.disable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Nextcloud disabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ocserv/enable")
|
||||
async def enable_ocserv():
|
||||
"""Enable Ocserv"""
|
||||
Ocserv.enable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Ocserv enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ocserv/disable")
|
||||
async def disable_ocserv():
|
||||
"""Disable Ocserv"""
|
||||
Ocserv.disable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Ocserv disabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/pleroma/enable")
|
||||
async def enable_pleroma():
|
||||
"""Enable Pleroma"""
|
||||
Pleroma.enable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Pleroma enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/pleroma/disable")
|
||||
async def disable_pleroma():
|
||||
"""Disable Pleroma"""
|
||||
Pleroma.disable()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Pleroma disabled",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/restic/backup/list")
|
||||
async def get_restic_backup_list():
|
||||
restic = ResticController()
|
||||
return restic.snapshot_list
|
||||
|
||||
|
||||
@router.put("/restic/backup/create")
|
||||
async def create_restic_backup():
|
||||
restic = ResticController()
|
||||
if restic.state is ResticStates.NO_KEY:
|
||||
raise HTTPException(status_code=400, detail="Backup key not provided")
|
||||
if restic.state is ResticStates.INITIALIZING:
|
||||
raise HTTPException(status_code=400, detail="Backup is initializing")
|
||||
if restic.state is ResticStates.BACKING_UP:
|
||||
raise HTTPException(status_code=409, detail="Backup is already running")
|
||||
restic_tasks.start_backup()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Backup creation has started",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/restic/backup/status")
|
||||
async def get_restic_backup_status():
|
||||
restic = ResticController()
|
||||
|
||||
return {
|
||||
"status": restic.state.name,
|
||||
"progress": restic.progress,
|
||||
"error_message": restic.error_message,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/restic/backup/reload")
|
||||
async def reload_restic_backup():
|
||||
restic_tasks.load_snapshots()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Snapshots reload started",
|
||||
}
|
||||
|
||||
|
||||
class BackupRestoreInput(BaseModel):
|
||||
backupId: str
|
||||
|
||||
|
||||
@router.put("/restic/backup/restore")
|
||||
async def restore_restic_backup(backup: BackupRestoreInput):
|
||||
restic = ResticController()
|
||||
if restic.state is ResticStates.NO_KEY:
|
||||
raise HTTPException(status_code=400, detail="Backup key not provided")
|
||||
if restic.state is ResticStates.NOT_INITIALIZED:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Backups repository is not initialized"
|
||||
)
|
||||
if restic.state is ResticStates.BACKING_UP:
|
||||
raise HTTPException(status_code=409, detail="Backup is already running")
|
||||
if restic.state is ResticStates.INITIALIZING:
|
||||
raise HTTPException(status_code=400, detail="Repository is initializing")
|
||||
if restic.state is ResticStates.RESTORING:
|
||||
raise HTTPException(status_code=409, detail="Restore is already running")
|
||||
|
||||
for backup_item in restic.snapshot_list:
|
||||
if backup_item["short_id"] == backup.backupId:
|
||||
restic_tasks.restore_from_backup(backup.backupId)
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "Backup restoration procedure started",
|
||||
}
|
||||
|
||||
raise HTTPException(status_code=404, detail="Backup not found")
|
||||
|
||||
|
||||
class BackblazeConfigInput(BaseModel):
|
||||
accountId: str
|
||||
accountKey: str
|
||||
bucket: str
|
||||
|
||||
|
||||
@router.put("/restic/backblaze/config")
|
||||
async def set_backblaze_config(backblaze_config: BackblazeConfigInput):
|
||||
with WriteUserData() as data:
|
||||
if "backblaze" not in data:
|
||||
data["backblaze"] = {}
|
||||
data["backblaze"]["accountId"] = backblaze_config.accountId
|
||||
data["backblaze"]["accountKey"] = backblaze_config.accountKey
|
||||
data["backblaze"]["bucket"] = backblaze_config.bucket
|
||||
|
||||
restic_tasks.update_keys_from_userdata()
|
||||
|
||||
return "New Backblaze settings saved"
|
||||
|
||||
|
||||
@router.post("/ssh/enable")
|
||||
async def rest_enable_ssh():
|
||||
"""Enable SSH"""
|
||||
enable_ssh()
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "SSH enabled",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/ssh")
|
||||
async def rest_get_ssh():
|
||||
"""Get the SSH configuration"""
|
||||
settings = get_ssh_settings()
|
||||
return {
|
||||
"enable": settings.enable,
|
||||
"passwordAuthentication": settings.passwordAuthentication,
|
||||
}
|
||||
|
||||
|
||||
class SshConfigInput(BaseModel):
|
||||
enable: Optional[bool] = None
|
||||
passwordAuthentication: Optional[bool] = None
|
||||
|
||||
|
||||
@router.put("/ssh")
|
||||
async def rest_set_ssh(ssh_config: SshConfigInput):
|
||||
"""Set the SSH configuration"""
|
||||
set_ssh_settings(ssh_config.enable, ssh_config.passwordAuthentication)
|
||||
|
||||
return "SSH settings changed"
|
||||
|
||||
|
||||
class SshKeyInput(BaseModel):
|
||||
public_key: str
|
||||
|
||||
|
||||
@router.put("/ssh/key/send", status_code=201)
|
||||
async def rest_send_ssh_key(input: SshKeyInput):
|
||||
"""Send the SSH key"""
|
||||
try:
|
||||
create_ssh_key("root", input.public_key)
|
||||
except KeyAlreadyExists as error:
|
||||
raise HTTPException(status_code=409, detail="Key already exists") from error
|
||||
except InvalidPublicKey as error:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
|
||||
) from error
|
||||
|
||||
return {
|
||||
"status": 0,
|
||||
"message": "SSH key sent",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/ssh/keys/{username}")
|
||||
async def rest_get_ssh_keys(username: str):
|
||||
"""Get the SSH keys for a user"""
|
||||
user = get_user_by_username(username)
|
||||
if user is None:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
|
||||
return user.ssh_keys
|
||||
|
||||
|
||||
@router.post("/ssh/keys/{username}", status_code=201)
|
||||
async def rest_add_ssh_key(username: str, input: SshKeyInput):
|
||||
try:
|
||||
create_ssh_key(username, input.public_key)
|
||||
except KeyAlreadyExists as error:
|
||||
raise HTTPException(status_code=409, detail="Key already exists") from error
|
||||
except InvalidPublicKey as error:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid key type. Only ssh-ed25519 and ssh-rsa are supported",
|
||||
) from error
|
||||
except UserNotFound as error:
|
||||
raise HTTPException(status_code=404, detail="User not found") from error
|
||||
|
||||
return {
|
||||
"message": "New SSH key successfully written",
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/ssh/keys/{username}")
|
||||
async def rest_delete_ssh_key(username: str, input: SshKeyInput):
|
||||
try:
|
||||
remove_ssh_key(username, input.public_key)
|
||||
except KeyNotFound as error:
|
||||
raise HTTPException(status_code=404, detail="Key not found") from error
|
||||
except UserNotFound as error:
|
||||
raise HTTPException(status_code=404, detail="User not found") from error
|
||||
return {"message": "SSH key deleted"}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
from typing import Optional
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from selfprivacy_api.dependencies import get_token_header
|
||||
|
||||
import selfprivacy_api.actions.system as system_actions
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/system",
|
||||
tags=["system"],
|
||||
dependencies=[Depends(get_token_header)],
|
||||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/configuration/timezone")
|
||||
async def get_timezone():
|
||||
"""Get the timezone of the server"""
|
||||
return system_actions.get_timezone()
|
||||
|
||||
|
||||
class ChangeTimezoneRequestBody(BaseModel):
|
||||
"""Change the timezone of the server"""
|
||||
|
||||
timezone: str
|
||||
|
||||
|
||||
@router.put("/configuration/timezone")
|
||||
async def change_timezone(timezone: ChangeTimezoneRequestBody):
|
||||
"""Change the timezone of the server"""
|
||||
try:
|
||||
system_actions.change_timezone(timezone.timezone)
|
||||
except system_actions.InvalidTimezone as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return {"timezone": timezone.timezone}
|
||||
|
||||
|
||||
@router.get("/configuration/autoUpgrade")
|
||||
async def get_auto_upgrade_settings():
|
||||
"""Get the auto-upgrade settings"""
|
||||
return system_actions.get_auto_upgrade_settings().dict()
|
||||
|
||||
|
||||
class AutoUpgradeSettings(BaseModel):
|
||||
"""Settings for auto-upgrading user data"""
|
||||
|
||||
enable: Optional[bool] = None
|
||||
allowReboot: Optional[bool] = None
|
||||
|
||||
|
||||
@router.put("/configuration/autoUpgrade")
|
||||
async def set_auto_upgrade_settings(settings: AutoUpgradeSettings):
|
||||
"""Set the auto-upgrade settings"""
|
||||
system_actions.set_auto_upgrade_settings(settings.enable, settings.allowReboot)
|
||||
return "Auto-upgrade settings changed"
|
||||
|
||||
|
||||
@router.get("/configuration/apply")
|
||||
async def apply_configuration():
|
||||
"""Apply the configuration"""
|
||||
return_code = system_actions.rebuild_system()
|
||||
return return_code
|
||||
|
||||
|
||||
@router.get("/configuration/rollback")
|
||||
async def rollback_configuration():
|
||||
"""Rollback the configuration"""
|
||||
return_code = system_actions.rollback_system()
|
||||
return return_code
|
||||
|
||||
|
||||
@router.get("/configuration/upgrade")
|
||||
async def upgrade_configuration():
|
||||
"""Upgrade the configuration"""
|
||||
return_code = system_actions.upgrade_system()
|
||||
return return_code
|
||||
|
||||
|
||||
@router.get("/reboot")
|
||||
async def reboot_system():
|
||||
"""Reboot the system"""
|
||||
system_actions.reboot_system()
|
||||
return "System reboot has started"
|
||||
|
||||
|
||||
@router.get("/version")
|
||||
async def get_system_version():
|
||||
"""Get the system version"""
|
||||
return {"system_version": system_actions.get_system_version()}
|
||||
|
||||
|
||||
@router.get("/pythonVersion")
|
||||
async def get_python_version():
|
||||
"""Get the Python version"""
|
||||
return system_actions.get_python_version()
|
||||
|
||||
|
||||
@router.get("/configuration/pull")
|
||||
async def pull_configuration():
|
||||
"""Pull the configuration"""
|
||||
action_result = system_actions.pull_repository_changes()
|
||||
if action_result.status == 0:
|
||||
return action_result.dict()
|
||||
raise HTTPException(status_code=500, detail=action_result.dict())
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
"""Users management module"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
import selfprivacy_api.actions.users as users_actions
|
||||
|
||||
from selfprivacy_api.dependencies import get_token_header
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/users",
|
||||
tags=["users"],
|
||||
dependencies=[Depends(get_token_header)],
|
||||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_users(withMainUser: bool = False):
|
||||
"""Get the list of users"""
|
||||
users: list[users_actions.UserDataUser] = users_actions.get_users(
|
||||
exclude_primary=not withMainUser, exclude_root=True
|
||||
)
|
||||
|
||||
return [user.username for user in users]
|
||||
|
||||
|
||||
class UserInput(BaseModel):
|
||||
"""User input"""
|
||||
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_user(user: UserInput):
|
||||
try:
|
||||
users_actions.create_user(user.username, user.password)
|
||||
except users_actions.PasswordIsEmpty as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except users_actions.UsernameForbidden as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
except users_actions.UsernameNotAlphanumeric as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except users_actions.UsernameTooLong as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except users_actions.UserAlreadyExists as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
return {"result": 0, "username": user.username}
|
||||
|
||||
|
||||
@router.delete("/{username}")
|
||||
async def delete_user(username: str):
|
||||
try:
|
||||
users_actions.delete_user(username)
|
||||
except users_actions.UserNotFound as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except users_actions.UserIsProtected as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
return {"result": 0, "username": username}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
"""Restic singleton controller."""
|
||||
from datetime import datetime
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
from threading import Lock
|
||||
from enum import Enum
|
||||
import portalocker
|
||||
from selfprivacy_api.utils import ReadUserData
|
||||
|
||||
|
||||
class ResticStates(Enum):
|
||||
"""Restic states enum."""
|
||||
|
||||
NO_KEY = 0
|
||||
NOT_INITIALIZED = 1
|
||||
INITIALIZED = 2
|
||||
BACKING_UP = 3
|
||||
RESTORING = 4
|
||||
ERROR = 5
|
||||
INITIALIZING = 6
|
||||
|
||||
|
||||
class ResticController:
|
||||
"""
|
||||
States in wich the restic_controller may be
|
||||
- no backblaze key
|
||||
- backblaze key is provided, but repository is not initialized
|
||||
- backblaze key is provided, repository is initialized
|
||||
- fetching list of snapshots
|
||||
- creating snapshot, current progress can be retrieved
|
||||
- recovering from snapshot
|
||||
|
||||
Any ongoing operation acquires the lock
|
||||
Current state can be fetched with get_state()
|
||||
"""
|
||||
|
||||
_instance = None
|
||||
_lock = Lock()
|
||||
_initialized = False
|
||||
|
||||
def __new__(cls):
|
||||
if not cls._instance:
|
||||
with cls._lock:
|
||||
cls._instance = super(ResticController, cls).__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self.state = ResticStates.NO_KEY
|
||||
self.lock = False
|
||||
self.progress = 0
|
||||
self._backblaze_account = None
|
||||
self._backblaze_key = None
|
||||
self._repository_name = None
|
||||
self.snapshot_list = []
|
||||
self.error_message = None
|
||||
self._initialized = True
|
||||
self.load_configuration()
|
||||
self.write_rclone_config()
|
||||
self.load_snapshots()
|
||||
|
||||
def load_configuration(self):
|
||||
"""Load current configuration from user data to singleton."""
|
||||
with ReadUserData() as user_data:
|
||||
self._backblaze_account = user_data["backblaze"]["accountId"]
|
||||
self._backblaze_key = user_data["backblaze"]["accountKey"]
|
||||
self._repository_name = user_data["backblaze"]["bucket"]
|
||||
if self._backblaze_account and self._backblaze_key and self._repository_name:
|
||||
self.state = ResticStates.INITIALIZING
|
||||
else:
|
||||
self.state = ResticStates.NO_KEY
|
||||
|
||||
def write_rclone_config(self):
|
||||
"""
|
||||
Open /root/.config/rclone/rclone.conf with portalocker
|
||||
and write configuration in the following format:
|
||||
[backblaze]
|
||||
type = b2
|
||||
account = {self.backblaze_account}
|
||||
key = {self.backblaze_key}
|
||||
"""
|
||||
with portalocker.Lock(
|
||||
"/root/.config/rclone/rclone.conf", "w", timeout=None
|
||||
) as rclone_config:
|
||||
rclone_config.write(
|
||||
f"[backblaze]\n"
|
||||
f"type = b2\n"
|
||||
f"account = {self._backblaze_account}\n"
|
||||
f"key = {self._backblaze_key}\n"
|
||||
)
|
||||
|
||||
def load_snapshots(self):
|
||||
"""
|
||||
Load list of snapshots from repository
|
||||
"""
|
||||
backup_listing_command = [
|
||||
"restic",
|
||||
"-o",
|
||||
"rclone.args=serve restic --stdio",
|
||||
"-r",
|
||||
f"rclone:backblaze:{self._repository_name}/sfbackup",
|
||||
"snapshots",
|
||||
"--json",
|
||||
]
|
||||
|
||||
if self.state in (ResticStates.BACKING_UP, ResticStates.RESTORING):
|
||||
return
|
||||
with subprocess.Popen(
|
||||
backup_listing_command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as backup_listing_process_descriptor:
|
||||
snapshots_list = backup_listing_process_descriptor.communicate()[0].decode(
|
||||
"utf-8"
|
||||
)
|
||||
try:
|
||||
starting_index = snapshots_list.find("[")
|
||||
json.loads(snapshots_list[starting_index:])
|
||||
self.snapshot_list = json.loads(snapshots_list[starting_index:])
|
||||
self.state = ResticStates.INITIALIZED
|
||||
print(snapshots_list)
|
||||
except ValueError:
|
||||
if "Is there a repository at the following location?" in snapshots_list:
|
||||
self.state = ResticStates.NOT_INITIALIZED
|
||||
return
|
||||
self.state = ResticStates.ERROR
|
||||
self.error_message = snapshots_list
|
||||
return
|
||||
|
||||
def initialize_repository(self):
|
||||
"""
|
||||
Initialize repository with restic
|
||||
"""
|
||||
initialize_repository_command = [
|
||||
"restic",
|
||||
"-o",
|
||||
"rclone.args=serve restic --stdio",
|
||||
"-r",
|
||||
f"rclone:backblaze:{self._repository_name}/sfbackup",
|
||||
"init",
|
||||
]
|
||||
with subprocess.Popen(
|
||||
initialize_repository_command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as initialize_repository_process_descriptor:
|
||||
msg = initialize_repository_process_descriptor.communicate()[0].decode(
|
||||
"utf-8"
|
||||
)
|
||||
if initialize_repository_process_descriptor.returncode == 0:
|
||||
self.state = ResticStates.INITIALIZED
|
||||
else:
|
||||
self.state = ResticStates.ERROR
|
||||
self.error_message = msg
|
||||
|
||||
self.state = ResticStates.INITIALIZED
|
||||
|
||||
def start_backup(self):
|
||||
"""
|
||||
Start backup with restic
|
||||
"""
|
||||
backup_command = [
|
||||
"restic",
|
||||
"-o",
|
||||
"rclone.args=serve restic --stdio",
|
||||
"-r",
|
||||
f"rclone:backblaze:{self._repository_name}/sfbackup",
|
||||
"--verbose",
|
||||
"--json",
|
||||
"backup",
|
||||
"/var",
|
||||
]
|
||||
with open("/var/backup.log", "w", encoding="utf-8") as log_file:
|
||||
subprocess.Popen(
|
||||
backup_command,
|
||||
shell=False,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
self.state = ResticStates.BACKING_UP
|
||||
self.progress = 0
|
||||
|
||||
def check_progress(self):
|
||||
"""
|
||||
Check progress of ongoing backup operation
|
||||
"""
|
||||
backup_status_check_command = ["tail", "-1", "/var/backup.log"]
|
||||
|
||||
if self.state in (ResticStates.NO_KEY, ResticStates.NOT_INITIALIZED):
|
||||
return
|
||||
|
||||
# If the log file does not exists
|
||||
if os.path.exists("/var/backup.log") is False:
|
||||
self.state = ResticStates.INITIALIZED
|
||||
|
||||
with subprocess.Popen(
|
||||
backup_status_check_command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
) as backup_status_check_process_descriptor:
|
||||
backup_process_status = (
|
||||
backup_status_check_process_descriptor.communicate()[0].decode("utf-8")
|
||||
)
|
||||
|
||||
try:
|
||||
status = json.loads(backup_process_status)
|
||||
except ValueError:
|
||||
print(backup_process_status)
|
||||
self.error_message = backup_process_status
|
||||
return
|
||||
if status["message_type"] == "status":
|
||||
self.progress = status["percent_done"]
|
||||
self.state = ResticStates.BACKING_UP
|
||||
elif status["message_type"] == "summary":
|
||||
self.state = ResticStates.INITIALIZED
|
||||
self.progress = 0
|
||||
self.snapshot_list.append(
|
||||
{
|
||||
"short_id": status["snapshot_id"],
|
||||
# Current time in format 2021-12-02T00:02:51.086452543+03:00
|
||||
"time": datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f%z"),
|
||||
}
|
||||
)
|
||||
|
||||
def restore_from_backup(self, snapshot_id):
|
||||
"""
|
||||
Restore from backup with restic
|
||||
"""
|
||||
backup_restoration_command = [
|
||||
"restic",
|
||||
"-o",
|
||||
"rclone.args=serve restic --stdio",
|
||||
"-r",
|
||||
f"rclone:backblaze:{self._repository_name}/sfbackup",
|
||||
"restore",
|
||||
snapshot_id,
|
||||
"--target",
|
||||
"/",
|
||||
]
|
||||
|
||||
self.state = ResticStates.RESTORING
|
||||
|
||||
subprocess.run(backup_restoration_command, shell=False)
|
||||
|
||||
self.state = ResticStates.INITIALIZED
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
"""Tasks for the restic controller."""
|
||||
from huey import crontab
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
from . import ResticController, ResticStates
|
||||
|
||||
|
||||
@huey.task()
|
||||
def init_restic():
|
||||
controller = ResticController()
|
||||
if controller.state == ResticStates.NOT_INITIALIZED:
|
||||
initialize_repository()
|
||||
|
||||
|
||||
@huey.task()
|
||||
def update_keys_from_userdata():
|
||||
controller = ResticController()
|
||||
controller.load_configuration()
|
||||
controller.write_rclone_config()
|
||||
initialize_repository()
|
||||
|
||||
|
||||
# Check every morning at 5:00 AM
|
||||
@huey.task(crontab(hour=5, minute=0))
|
||||
def cron_load_snapshots():
|
||||
controller = ResticController()
|
||||
controller.load_snapshots()
|
||||
|
||||
|
||||
# Check every morning at 5:00 AM
|
||||
@huey.task()
|
||||
def load_snapshots():
|
||||
controller = ResticController()
|
||||
controller.load_snapshots()
|
||||
if controller.state == ResticStates.NOT_INITIALIZED:
|
||||
load_snapshots.schedule(delay=120)
|
||||
|
||||
|
||||
@huey.task()
|
||||
def initialize_repository():
|
||||
controller = ResticController()
|
||||
if controller.state is not ResticStates.NO_KEY:
|
||||
controller.initialize_repository()
|
||||
load_snapshots()
|
||||
|
||||
|
||||
@huey.task()
|
||||
def fetch_backup_status():
|
||||
controller = ResticController()
|
||||
if controller.state is ResticStates.BACKING_UP:
|
||||
controller.check_progress()
|
||||
if controller.state is ResticStates.BACKING_UP:
|
||||
fetch_backup_status.schedule(delay=2)
|
||||
else:
|
||||
load_snapshots.schedule(delay=240)
|
||||
|
||||
|
||||
@huey.task()
|
||||
def start_backup():
|
||||
controller = ResticController()
|
||||
if controller.state is ResticStates.NOT_INITIALIZED:
|
||||
resp = initialize_repository()
|
||||
resp.get()
|
||||
controller.start_backup()
|
||||
fetch_backup_status.schedule(delay=3)
|
||||
|
||||
|
||||
@huey.task()
|
||||
def restore_from_backup(snapshot):
|
||||
controller = ResticController()
|
||||
controller.restore_from_backup(snapshot)
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
import typing
|
||||
from selfprivacy_api.services.bitwarden import Bitwarden
|
||||
from selfprivacy_api.services.gitea import Gitea
|
||||
from selfprivacy_api.services.jitsimeet import JitsiMeet
|
||||
from selfprivacy_api.services.jitsi import Jitsi
|
||||
from selfprivacy_api.services.mailserver import MailServer
|
||||
from selfprivacy_api.services.nextcloud import Nextcloud
|
||||
from selfprivacy_api.services.pleroma import Pleroma
|
||||
|
|
@ -18,7 +18,7 @@ services: list[Service] = [
|
|||
Nextcloud(),
|
||||
Pleroma(),
|
||||
Ocserv(),
|
||||
JitsiMeet(),
|
||||
Jitsi(),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ def get_disabled_services() -> list[Service]:
|
|||
|
||||
|
||||
def get_services_by_location(location: str) -> list[Service]:
|
||||
return [service for service in services if service.get_drive() == location]
|
||||
return [service for service in services if service.get_location() == location]
|
||||
|
||||
|
||||
def get_all_required_dns_records() -> list[ServiceDnsRecord]:
|
||||
|
|
@ -54,20 +54,14 @@ def get_all_required_dns_records() -> list[ServiceDnsRecord]:
|
|||
name="api",
|
||||
content=ip4,
|
||||
ttl=3600,
|
||||
display_name="SelfPrivacy API",
|
||||
),
|
||||
ServiceDnsRecord(
|
||||
type="AAAA",
|
||||
name="api",
|
||||
content=ip6,
|
||||
ttl=3600,
|
||||
),
|
||||
]
|
||||
|
||||
if ip6 is not None:
|
||||
dns_records.append(
|
||||
ServiceDnsRecord(
|
||||
type="AAAA",
|
||||
name="api",
|
||||
content=ip6,
|
||||
ttl=3600,
|
||||
display_name="SelfPrivacy API (IPv6)",
|
||||
)
|
||||
)
|
||||
for service in get_enabled_services():
|
||||
dns_records += service.get_dns_records(ip4, ip6)
|
||||
dns_records += service.get_dns_records()
|
||||
return dns_records
|
||||
|
|
|
|||
|
|
@ -1,12 +1,17 @@
|
|||
"""Class representing Bitwarden service"""
|
||||
import base64
|
||||
import subprocess
|
||||
from typing import Optional, List
|
||||
import typing
|
||||
|
||||
from selfprivacy_api.utils import get_domain
|
||||
|
||||
from selfprivacy_api.utils.systemd import get_service_status
|
||||
from selfprivacy_api.services.service import Service, ServiceStatus
|
||||
from selfprivacy_api.jobs import Job, JobStatus, Jobs
|
||||
from selfprivacy_api.services.generic_service_mover import FolderMoveNames, move_service
|
||||
from selfprivacy_api.services.generic_size_counter import get_storage_usage
|
||||
from selfprivacy_api.services.generic_status_getter import get_service_status
|
||||
from selfprivacy_api.services.service import Service, ServiceDnsRecord, ServiceStatus
|
||||
from selfprivacy_api.utils import ReadUserData, WriteUserData, get_domain
|
||||
from selfprivacy_api.utils.block_devices import BlockDevice
|
||||
from selfprivacy_api.utils.huey import huey
|
||||
import selfprivacy_api.utils.network as network_utils
|
||||
from selfprivacy_api.services.bitwarden.icon import BITWARDEN_ICON
|
||||
|
||||
|
||||
|
|
@ -34,19 +39,11 @@ class Bitwarden(Service):
|
|||
return base64.b64encode(BITWARDEN_ICON.encode("utf-8")).decode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def get_user() -> str:
|
||||
return "vaultwarden"
|
||||
|
||||
@staticmethod
|
||||
def get_url() -> Optional[str]:
|
||||
def get_url() -> typing.Optional[str]:
|
||||
"""Return service url."""
|
||||
domain = get_domain()
|
||||
return f"https://password.{domain}"
|
||||
|
||||
@staticmethod
|
||||
def get_subdomain() -> Optional[str]:
|
||||
return "password"
|
||||
|
||||
@staticmethod
|
||||
def is_movable() -> bool:
|
||||
return True
|
||||
|
|
@ -56,8 +53,9 @@ class Bitwarden(Service):
|
|||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_backup_description() -> str:
|
||||
return "Password database, encryption certificate and attachments."
|
||||
def is_enabled() -> bool:
|
||||
with ReadUserData() as user_data:
|
||||
return user_data.get("bitwarden", {}).get("enable", False)
|
||||
|
||||
@staticmethod
|
||||
def get_status() -> ServiceStatus:
|
||||
|
|
@ -72,6 +70,22 @@ class Bitwarden(Service):
|
|||
"""
|
||||
return get_service_status("vaultwarden.service")
|
||||
|
||||
@staticmethod
|
||||
def enable():
|
||||
"""Enable Bitwarden service."""
|
||||
with WriteUserData() as user_data:
|
||||
if "bitwarden" not in user_data:
|
||||
user_data["bitwarden"] = {}
|
||||
user_data["bitwarden"]["enable"] = True
|
||||
|
||||
@staticmethod
|
||||
def disable():
|
||||
"""Disable Bitwarden service."""
|
||||
with WriteUserData() as user_data:
|
||||
if "bitwarden" not in user_data:
|
||||
user_data["bitwarden"] = {}
|
||||
user_data["bitwarden"]["enable"] = False
|
||||
|
||||
@staticmethod
|
||||
def stop():
|
||||
subprocess.run(["systemctl", "stop", "vaultwarden.service"])
|
||||
|
|
@ -97,5 +111,64 @@ class Bitwarden(Service):
|
|||
return ""
|
||||
|
||||
@staticmethod
|
||||
def get_folders() -> List[str]:
|
||||
return ["/var/lib/bitwarden", "/var/lib/bitwarden_rs"]
|
||||
def get_storage_usage() -> int:
|
||||
storage_usage = 0
|
||||
storage_usage += get_storage_usage("/var/lib/bitwarden")
|
||||
storage_usage += get_storage_usage("/var/lib/bitwarden_rs")
|
||||
return storage_usage
|
||||
|
||||
@staticmethod
|
||||
def get_location() -> str:
|
||||
with ReadUserData() as user_data:
|
||||
if user_data.get("useBinds", False):
|
||||
return user_data.get("bitwarden", {}).get("location", "sda1")
|
||||
else:
|
||||
return "sda1"
|
||||
|
||||
@staticmethod
|
||||
def get_dns_records() -> typing.List[ServiceDnsRecord]:
|
||||
"""Return list of DNS records for Bitwarden service."""
|
||||
return [
|
||||
ServiceDnsRecord(
|
||||
type="A",
|
||||
name="password",
|
||||
content=network_utils.get_ip4(),
|
||||
ttl=3600,
|
||||
),
|
||||
ServiceDnsRecord(
|
||||
type="AAAA",
|
||||
name="password",
|
||||
content=network_utils.get_ip6(),
|
||||
ttl=3600,
|
||||
),
|
||||
]
|
||||
|
||||
def move_to_volume(self, volume: BlockDevice) -> Job:
|
||||
job = Jobs.get_instance().add(
|
||||
type_id="services.bitwarden.move",
|
||||
name="Move Bitwarden",
|
||||
description=f"Moving Bitwarden data to {volume.name}",
|
||||
)
|
||||
|
||||
move_service(
|
||||
self,
|
||||
volume,
|
||||
job,
|
||||
[
|
||||
FolderMoveNames(
|
||||
name="bitwarden",
|
||||
bind_location="/var/lib/bitwarden",
|
||||
group="vaultwarden",
|
||||
owner="vaultwarden",
|
||||
),
|
||||
FolderMoveNames(
|
||||
name="bitwarden_rs",
|
||||
bind_location="/var/lib/bitwarden_rs",
|
||||
group="vaultwarden",
|
||||
owner="vaultwarden",
|
||||
),
|
||||
],
|
||||
"bitwarden",
|
||||
)
|
||||
|
||||
return job
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue