From b4fe79fdcebffacb96febe2e48f27c61ef07ed32 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 12 Feb 2024 19:58:24 +0400 Subject: [PATCH 1/4] devshell: add black, mypy and corresponding python-lsp-server plugins --- flake.nix | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index f1a317a..1dc3d80 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,9 @@ let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; + pythonPackages = pkgs.python310Packages; selfprivacy-graphql-api = pkgs.callPackage ./default.nix { - pythonPackages = pkgs.python310Packages; + inherit pythonPackages; rev = self.shortRev or self.dirtyShortRev or "dirty"; }; python = self.packages.${system}.default.pythonModule; @@ -71,15 +72,26 @@ import ./nixos/module.nix self.packages.${system}.default; devShells.${system}.default = pkgs.mkShell { name = "SP API dev shell"; - packages = with pkgs; [ - python-env - rclone - redis - restic - self.packages.${system}.pytest-vm - # FIXME consider loading this explicitly only after ArchLinux issue is solved - self.checks.x86_64-linux.default.driverInteractive - ]; + packages = with pythonPackages; + [ + black + mypy + pylsp-mypy + python-lsp-black + python-lsp-server + ] ++ + (with pkgs; + [ + nixpkgs-fmt + rclone + redis + restic + self.packages.${system}.pytest-vm + # FIXME consider loading this explicitly only after ArchLinux issue is solved + self.checks.x86_64-linux.default.driverInteractive + # the target API application python environment + python-env + ]); shellHook = '' # envs set with export and as attributes are treated differently. # for example. printenv will not fetch the value of an attribute. From ad6514658da80cf1df9483fddd78a61e805a6988 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 13 Feb 2024 01:47:58 +0400 Subject: [PATCH 2/4] fix devshell: Strawberry CLI dependencies --- flake.nix | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index 1dc3d80..f82fcf5 100644 --- a/flake.nix +++ b/flake.nix @@ -7,9 +7,8 @@ let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; - pythonPackages = pkgs.python310Packages; selfprivacy-graphql-api = pkgs.callPackage ./default.nix { - inherit pythonPackages; + pythonPackages = pkgs.python310Packages; rev = self.shortRev or self.dirtyShortRev or "dirty"; }; python = self.packages.${system}.default.pythonModule; @@ -20,7 +19,14 @@ pytest pytest-datadir pytest-mock - ])); + black + mypy + pylsp-mypy + python-lsp-black + python-lsp-server + typer # for strawberry + ] ++ strawberry-graphql.optional-dependencies.cli)); + vmtest-src-dir = "/root/source"; shellMOTD = '' Welcome to SP API development shell! @@ -72,26 +78,17 @@ import ./nixos/module.nix self.packages.${system}.default; devShells.${system}.default = pkgs.mkShell { name = "SP API dev shell"; - packages = with pythonPackages; - [ - black - mypy - pylsp-mypy - python-lsp-black - python-lsp-server - ] ++ - (with pkgs; - [ - nixpkgs-fmt - rclone - redis - restic - self.packages.${system}.pytest-vm - # FIXME consider loading this explicitly only after ArchLinux issue is solved - self.checks.x86_64-linux.default.driverInteractive - # the target API application python environment - python-env - ]); + packages = with pkgs; [ + nixpkgs-fmt + rclone + redis + restic + self.packages.${system}.pytest-vm + # FIXME consider loading this explicitly only after ArchLinux issue is solved + self.checks.x86_64-linux.default.driverInteractive + # the target API application python environment + python-env + ]; shellHook = '' # envs set with export and as attributes are treated differently. # for example. printenv will not fetch the value of an attribute. From bc0602bfcb52dd9da8d61cac7b093f07ffff33d0 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 12 Feb 2024 20:47:33 +0000 Subject: [PATCH 3/4] fix(backup): rework caching so that there are rarer api calls --- selfprivacy_api/backup/__init__.py | 35 +++++++++++++++++------------- selfprivacy_api/backup/tasks.py | 2 +- tests/test_backup.py | 14 ++++++++---- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 1c4c4e0..0a6d250 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -254,7 +254,7 @@ class Backups: reason=reason, ) - Backups._store_last_snapshot(service_name, snapshot) + Backups._on_new_snapshot_created(service_name, snapshot) if reason == BackupReason.AUTO: Backups._prune_auto_snaps(service) service.post_restore() @@ -265,7 +265,16 @@ class Backups: Jobs.update(job, status=JobStatus.FINISHED) if reason in [BackupReason.AUTO, BackupReason.PRE_RESTORE]: Jobs.set_expiration(job, AUTOBACKUP_JOB_EXPIRATION_SECONDS) - return snapshot + return Backups.sync_date_from_cache(snapshot) + + @staticmethod + def sync_date_from_cache(snapshot: Snapshot) -> Snapshot: + """ + Our snapshot creation dates are different from those on server by a tiny amount. + This is a convenience, maybe it is better to write a special comparison + function for snapshots + """ + return Storage.get_cached_snapshot_by_id(snapshot.id) @staticmethod def _auto_snaps(service): @@ -523,13 +532,12 @@ class Backups: @staticmethod def get_all_snapshots() -> List[Snapshot]: """Returns all snapshots""" - cached_snapshots = Storage.get_cached_snapshots() - if cached_snapshots: - return cached_snapshots - # TODO: the oldest snapshots will get expired faster than the new ones. - # How to detect that the end is missing? + # When we refresh our cache: + # 1. Manually + # 2. On timer + # 3. On new snapshot + # 4. On snapshot deletion - Backups.force_snapshot_cache_reload() return Storage.get_cached_snapshots() @staticmethod @@ -548,15 +556,13 @@ class Backups: @staticmethod def forget_snapshots(snapshots: List[Snapshot]) -> None: """ - Deletes a batch of snapshots from the repo and from cache + Deletes a batch of snapshots from the repo and syncs cache Optimized """ ids = [snapshot.id for snapshot in snapshots] Backups.provider().backupper.forget_snapshots(ids) - # less critical - for snapshot in snapshots: - Storage.delete_cached_snapshot(snapshot) + Backups.force_snapshot_cache_reload() @staticmethod def forget_snapshot(snapshot: Snapshot) -> None: @@ -593,12 +599,11 @@ class Backups: ) @staticmethod - def _store_last_snapshot(service_id: str, snapshot: Snapshot) -> None: + def _on_new_snapshot_created(service_id: str, snapshot: Snapshot) -> None: """What do we do with a snapshot that is just made?""" # non-expiring timestamp of the last Storage.store_last_timestamp(service_id, snapshot) - # expiring cache entry - Storage.cache_snapshot(snapshot) + Backups.force_snapshot_cache_reload() # Autobackup diff --git a/selfprivacy_api/backup/tasks.py b/selfprivacy_api/backup/tasks.py index 6520c70..465b1a8 100644 --- a/selfprivacy_api/backup/tasks.py +++ b/selfprivacy_api/backup/tasks.py @@ -82,6 +82,6 @@ def automatic_backup(): start_backup(service, BackupReason.AUTO) -@huey.periodic_task(crontab(hour=SNAPSHOT_CACHE_TTL_HOURS)) +@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS))) def reload_snapshot_cache(): Backups.force_snapshot_cache_reload() diff --git a/tests/test_backup.py b/tests/test_backup.py index 23569eb..f2b34a8 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -165,7 +165,7 @@ def test_reinit_after_purge(backups): Backups.erase_repo() assert Backups.is_initted() is False with pytest.raises(ValueError): - Backups.get_all_snapshots() + Backups.force_snapshot_cache_reload() Backups.init_repo() assert Backups.is_initted() is True @@ -209,7 +209,11 @@ def test_backup_returns_snapshot(backups, dummy_service): snapshot = provider.backupper.start_backup(service_folders, name) assert snapshot.id is not None - assert len(snapshot.id) == len(Backups.get_all_snapshots()[0].id) + + snapshots = provider.backupper.get_snapshots() + assert snapshots != [] + + assert len(snapshot.id) == len(snapshots[0].id) assert Backups.get_snapshot_by_id(snapshot.id) is not None assert snapshot.service_name == name assert snapshot.created_at is not None @@ -472,10 +476,12 @@ def test_snapshots_caching(backups, dummy_service): cached_snapshots = Storage.get_cached_snapshots() assert len(cached_snapshots) == 0 + # We do not assume that no snapshots means we need to reload the cache snapshots = Backups.get_snapshots(dummy_service) - assert len(snapshots) == 1 + assert len(snapshots) == 0 + # No cache reload happened cached_snapshots = Storage.get_cached_snapshots() - assert len(cached_snapshots) == 1 + assert len(cached_snapshots) == 0 def lowlevel_forget(snapshot_id): From 4757bedc4ec62d3577fd1f259abbe34ba6dce893 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 12 Feb 2024 21:23:01 +0000 Subject: [PATCH 4/4] test(backup): add tests for cache reloads --- selfprivacy_api/backup/__init__.py | 3 +-- tests/test_backup.py | 32 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 0a6d250..e4b5db7 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -567,8 +567,7 @@ class Backups: @staticmethod def forget_snapshot(snapshot: Snapshot) -> None: """Deletes a snapshot from the repo and from cache""" - Backups.provider().backupper.forget_snapshot(snapshot.id) - Storage.delete_cached_snapshot(snapshot) + Backups.forget_snapshots([snapshot]) @staticmethod def forget_all_snapshots(): diff --git a/tests/test_backup.py b/tests/test_backup.py index f2b34a8..0a2b3ed 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -472,7 +472,8 @@ def test_snapshots_caching(backups, dummy_service): cached_snapshots = Storage.get_cached_snapshots() assert len(cached_snapshots) == 1 - Storage.delete_cached_snapshot(cached_snapshots[0]) + snap_to_uncache = cached_snapshots[0] + Storage.delete_cached_snapshot(snap_to_uncache) cached_snapshots = Storage.get_cached_snapshots() assert len(cached_snapshots) == 0 @@ -484,6 +485,35 @@ def test_snapshots_caching(backups, dummy_service): assert len(cached_snapshots) == 0 +# Storage +def test_snapshot_cache_autoreloads(backups, dummy_service): + Backups.back_up(dummy_service) + + cached_snapshots = Storage.get_cached_snapshots() + assert len(cached_snapshots) == 1 + snap_to_uncache = cached_snapshots[0] + + Storage.delete_cached_snapshot(snap_to_uncache) + cached_snapshots = Storage.get_cached_snapshots() + assert len(cached_snapshots) == 0 + + # When we create a snapshot we do reload cache + Backups.back_up(dummy_service) + cached_snapshots = Storage.get_cached_snapshots() + assert len(cached_snapshots) == 2 + assert snap_to_uncache in cached_snapshots + + Storage.delete_cached_snapshot(snap_to_uncache) + cached_snapshots = Storage.get_cached_snapshots() + assert len(cached_snapshots) == 1 + + # When we try to delete a snapshot we cannot find in cache, it is ok and we do reload cache + Backups.forget_snapshot(snap_to_uncache) + cached_snapshots = Storage.get_cached_snapshots() + assert len(cached_snapshots) == 1 + assert snap_to_uncache not in cached_snapshots + + def lowlevel_forget(snapshot_id): Backups.provider().backupper.forget_snapshot(snapshot_id)