diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 23e706f..bd16488 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -1,3 +1,6 @@ +""" +This module contains the controller class for backups. +""" from datetime import datetime, timedelta from os import statvfs from typing import List, Optional @@ -42,8 +45,12 @@ DEFAULT_JSON_PROVIDER = { class NotDeadError(AssertionError): + """ + This error is raised when we try to back up a service that is not dead yet. + """ def __init__(self, service: Service): self.service_name = service.get_id() + super().__init__() def __str__(self): return f""" @@ -61,6 +68,9 @@ class Backups: @staticmethod def provider() -> AbstractBackupProvider: + """ + Returns the current backup storage provider. + """ return Backups._lookup_provider() @staticmethod @@ -71,6 +81,13 @@ class Backups: location: str, repo_id: str = "", ) -> None: + """ + Sets the new configuration of the backup storage provider. + + In case of `BackupProviderEnum.BACKBLAZE`, the `login` is the key ID, + the `key` is the key itself, and the `location` is the bucket name and + the `repo_id` is the bucket ID. + """ provider: AbstractBackupProvider = Backups._construct_provider( kind, login, @@ -82,6 +99,9 @@ class Backups: @staticmethod def reset(reset_json=True) -> None: + """ + Deletes all the data about the backup storage provider. + """ Storage.reset() if reset_json: try: @@ -183,11 +203,19 @@ class Backups: @staticmethod def init_repo() -> None: + """ + Initializes the backup repository. This is required once per repo. + """ Backups.provider().backupper.init() Storage.mark_as_init() @staticmethod def is_initted() -> bool: + """ + Returns whether the backup repository is initialized or not. + If it is not initialized, we cannot back up and probably should + call `init_repo` first. + """ if Storage.has_init_mark(): return True @@ -219,9 +247,9 @@ class Backups: ) Backups._store_last_snapshot(tag, snapshot) service.post_restore() - except Exception as e: + except Exception as error: Jobs.update(job, status=JobStatus.ERROR) - raise e + raise error Jobs.update(job, status=JobStatus.FINISHED) return snapshot @@ -252,16 +280,17 @@ class Backups: snapshot.id, verify=False, ) - except Exception as e: + except Exception as error: Backups._restore_service_from_snapshot( service, failsafe_snapshot.id, verify=False ) - raise e + raise error @staticmethod def restore_snapshot( snapshot: Snapshot, strategy=RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE ) -> None: + """Restores a snapshot to its original service using the given strategy""" service = get_service_by_id(snapshot.service_name) if service is None: raise ValueError( @@ -283,9 +312,9 @@ class Backups: service.post_restore() - except Exception as e: + except Exception as error: Jobs.update(job, status=JobStatus.ERROR) - raise e + raise error Jobs.update(job, status=JobStatus.FINISHED) @@ -338,6 +367,7 @@ class Backups: @staticmethod def get_snapshots(service: Service) -> List[Snapshot]: + """Returns all snapshots for a given service""" snapshots = Backups.get_all_snapshots() service_id = service.get_id() return list( @@ -349,8 +379,9 @@ class Backups: @staticmethod def get_all_snapshots() -> List[Snapshot]: + """Returns all snapshots""" cached_snapshots = Storage.get_cached_snapshots() - if cached_snapshots != []: + if cached_snapshots: return cached_snapshots # TODO: the oldest snapshots will get expired faster than the new ones. # How to detect that the end is missing? @@ -359,24 +390,32 @@ class Backups: return Storage.get_cached_snapshots() @staticmethod - def get_snapshot_by_id(id: str) -> Optional[Snapshot]: - snap = Storage.get_cached_snapshot_by_id(id) + def get_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]: + """Returns a backup snapshot by its id""" + snap = Storage.get_cached_snapshot_by_id(snapshot_id) if snap is not None: return snap # Possibly our cache entry got invalidated, let's try one more time Backups.force_snapshot_cache_reload() - snap = Storage.get_cached_snapshot_by_id(id) + snap = Storage.get_cached_snapshot_by_id(snapshot_id) return snap @staticmethod def forget_snapshot(snapshot: Snapshot) -> None: + """Deletes a snapshot from the storage""" Backups.provider().backupper.forget_snapshot(snapshot.id) Storage.delete_cached_snapshot(snapshot) @staticmethod def force_snapshot_cache_reload() -> None: + """ + Forces a reload of the snapshot cache. + + This may be an expensive operation, so use it wisely. + User pays for the API calls. + """ upstream_snapshots = Backups.provider().backupper.get_snapshots() Storage.invalidate_snapshot_storage() for snapshot in upstream_snapshots: @@ -384,6 +423,7 @@ class Backups: @staticmethod def snapshot_restored_size(snapshot_id: str) -> int: + """Returns the size of the snapshot""" return Backups.provider().backupper.restored_size( snapshot_id, ) @@ -434,6 +474,7 @@ class Backups: @staticmethod def services_to_back_up(time: datetime) -> List[Service]: + """Returns a list of services that should be backed up at a given time""" return [ service for service in get_all_services() @@ -447,6 +488,7 @@ class Backups: @staticmethod def is_time_to_backup_service(service: Service, time: datetime): + """Returns True if it is time to back up a service""" period = Backups.autobackup_period_minutes() service_id = service.get_id() if not service.can_be_backed_up(): @@ -467,6 +509,10 @@ class Backups: @staticmethod def space_usable_for_service(service: Service) -> int: + """ + Returns the amount of space available on the volume the given + service is located on. + """ folders = service.get_folders() if folders == []: raise ValueError("unallocated service", service.get_id()) @@ -478,6 +524,8 @@ class Backups: @staticmethod def set_localfile_repo(file_path: str): + """Used by tests to set a local folder as a backup repo""" + # pylint: disable-next=invalid-name ProviderClass = get_provider(BackupProviderEnum.FILE) provider = ProviderClass( login="", @@ -490,10 +538,7 @@ class Backups: @staticmethod def assert_dead(service: Service): """ - - If we backup the service that is failing to restore it to the previous snapshot, - its status can be FAILED. - And obviously restoring a failed service is the main route + Checks if a service is dead and can be safely restored from a snapshot. """ if service.get_status() not in [ ServiceStatus.INACTIVE, diff --git a/selfprivacy_api/backup/backuppers/__init__.py b/selfprivacy_api/backup/backuppers/__init__.py index 7051a6a..ea2350b 100644 --- a/selfprivacy_api/backup/backuppers/__init__.py +++ b/selfprivacy_api/backup/backuppers/__init__.py @@ -5,19 +5,25 @@ from selfprivacy_api.models.backup.snapshot import Snapshot class AbstractBackupper(ABC): + """Abstract class for backuppers""" + + # flake8: noqa: B027 def __init__(self) -> None: pass @abstractmethod def is_initted(self) -> bool: + """Returns true if the repository is initted""" raise NotImplementedError @abstractmethod def set_creds(self, account: str, key: str, repo: str) -> None: + """Set the credentials for the backupper""" raise NotImplementedError @abstractmethod - def start_backup(self, folders: List[str], repo_name: str) -> Snapshot: + def start_backup(self, folders: List[str], tag: str) -> Snapshot: + """Start a backup of the given folders""" raise NotImplementedError @abstractmethod @@ -27,6 +33,7 @@ class AbstractBackupper(ABC): @abstractmethod def init(self) -> None: + """Initialize the repository""" raise NotImplementedError @abstractmethod @@ -41,8 +48,10 @@ class AbstractBackupper(ABC): @abstractmethod def restored_size(self, snapshot_id: str) -> int: + """Get the size of the restored snapshot""" raise NotImplementedError @abstractmethod def forget_snapshot(self, snapshot_id) -> None: + """Forget a snapshot""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/none_backupper.py b/selfprivacy_api/backup/backuppers/none_backupper.py index 2ac2035..d9edaeb 100644 --- a/selfprivacy_api/backup/backuppers/none_backupper.py +++ b/selfprivacy_api/backup/backuppers/none_backupper.py @@ -5,13 +5,15 @@ from selfprivacy_api.backup.backuppers import AbstractBackupper class NoneBackupper(AbstractBackupper): + """A backupper that does nothing""" + def is_initted(self, repo_name: str = "") -> bool: return False def set_creds(self, account: str, key: str, repo: str): pass - def start_backup(self, folders: List[str], repo_name: str): + def start_backup(self, folders: List[str], tag: str): raise NotImplementedError def get_snapshots(self) -> List[Snapshot]: @@ -21,7 +23,7 @@ class NoneBackupper(AbstractBackupper): def init(self): raise NotImplementedError - def restore_from_backup(self, snapshot_id: str, folders: List[str]): + def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True): """Restore a target folder using a snapshot""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index 60e8285..b69c85d 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -21,13 +21,14 @@ from selfprivacy_api.backup.local_secret import LocalBackupSecret class ResticBackupper(AbstractBackupper): - def __init__(self, login_flag: str, key_flag: str, type: str) -> None: + def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None: self.login_flag = login_flag self.key_flag = key_flag - self.type = type + self.storage_type = storage_type self.account = "" self.key = "" self.repo = "" + super().__init__() def set_creds(self, account: str, key: str, repo: str) -> None: self.account = account @@ -37,7 +38,7 @@ class ResticBackupper(AbstractBackupper): def restic_repo(self) -> str: # https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone # https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5 - return f"rclone:{self.type}{self.repo}" + return f"rclone:{self.storage_type}{self.repo}" def rclone_args(self): return "rclone.args=serve restic --stdio " + self.backend_rclone_args() @@ -72,12 +73,12 @@ class ResticBackupper(AbstractBackupper): tag, ] ) - if args != []: + if args: command.extend(ResticBackupper.__flatten_list(args)) return command - def mount_repo(self, dir): - mount_command = self.restic_command("mount", dir) + def mount_repo(self, mount_directory): + mount_command = self.restic_command("mount", mount_directory) mount_command.insert(0, "nohup") handle = subprocess.Popen( mount_command, @@ -85,28 +86,28 @@ class ResticBackupper(AbstractBackupper): shell=False, ) sleep(2) - if "ids" not in listdir(dir): - raise IOError("failed to mount dir ", dir) + if "ids" not in listdir(mount_directory): + raise IOError("failed to mount dir ", mount_directory) return handle - def unmount_repo(self, dir): - mount_command = ["umount", "-l", dir] + def unmount_repo(self, mount_directory): + mount_command = ["umount", "-l", mount_directory] with subprocess.Popen( mount_command, stdout=subprocess.PIPE, shell=False ) as handle: output = handle.communicate()[0].decode("utf-8") # TODO: check for exit code? if "error" in output.lower(): - return IOError("failed to unmount dir ", dir, ": ", output) + return IOError("failed to unmount dir ", mount_directory, ": ", output) - if not listdir(dir) == []: - return IOError("failed to unmount dir ", dir) + if not listdir(mount_directory) == []: + return IOError("failed to unmount dir ", mount_directory) @staticmethod - def __flatten_list(list): + def __flatten_list(list_to_flatten): """string-aware list flattener""" result = [] - for item in list: + for item in list_to_flatten: if isinstance(item, Iterable) and not isinstance(item, str): result.extend(ResticBackupper.__flatten_list(item)) continue @@ -147,8 +148,8 @@ class ResticBackupper(AbstractBackupper): messages, tag, ) - except ValueError as e: - raise ValueError("Could not create a snapshot: ", messages) from e + except ValueError as error: + raise ValueError("Could not create a snapshot: ", messages) from error @staticmethod def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot: @@ -231,8 +232,8 @@ class ResticBackupper(AbstractBackupper): try: parsed_output = ResticBackupper.parse_json_output(output) return parsed_output["total_size"] - except ValueError as e: - raise ValueError("cannot restore a snapshot: " + output) from e + except ValueError as error: + raise ValueError("cannot restore a snapshot: " + output) from error def restore_from_backup( self, @@ -246,13 +247,13 @@ class ResticBackupper(AbstractBackupper): if folders is None or folders == []: raise ValueError("cannot restore without knowing where to!") - with tempfile.TemporaryDirectory() as dir: + with tempfile.TemporaryDirectory() as temp_dir: if verify: - self._raw_verified_restore(snapshot_id, target=dir) - snapshot_root = dir + self._raw_verified_restore(snapshot_id, target=temp_dir) + snapshot_root = temp_dir else: # attempting inplace restore via mount + sync - self.mount_repo(dir) - snapshot_root = join(dir, "ids", snapshot_id) + self.mount_repo(temp_dir) + snapshot_root = join(temp_dir, "ids", snapshot_id) assert snapshot_root is not None for folder in folders: @@ -263,7 +264,7 @@ class ResticBackupper(AbstractBackupper): sync(src, dst) if not verify: - self.unmount_repo(dir) + self.unmount_repo(temp_dir) def _raw_verified_restore(self, snapshot_id, target="/"): """barebones restic restore""" @@ -355,8 +356,8 @@ class ResticBackupper(AbstractBackupper): raise ValueError("No repository! : " + output) try: return ResticBackupper.parse_json_output(output) - except ValueError as e: - raise ValueError("Cannot load snapshots: ") from e + except ValueError as error: + raise ValueError("Cannot load snapshots: ") from error def get_snapshots(self) -> List[Snapshot]: """Get all snapshots from the repo""" @@ -383,10 +384,10 @@ class ResticBackupper(AbstractBackupper): if len(json_messages) == 1: try: return json.loads(truncated_output) - except JSONDecodeError as e: + except JSONDecodeError as error: raise ValueError( "There is no json in the restic output : " + output - ) from e + ) from error result_array = [] for message in json_messages: diff --git a/selfprivacy_api/backup/providers/none.py b/selfprivacy_api/backup/providers/none.py index 474d0a2..6a37771 100644 --- a/selfprivacy_api/backup/providers/none.py +++ b/selfprivacy_api/backup/providers/none.py @@ -1,4 +1,4 @@ -from .provider import AbstractBackupProvider +from selfprivacy_api.backup.providers.provider import AbstractBackupProvider from selfprivacy_api.backup.backuppers.none_backupper import NoneBackupper from selfprivacy_api.graphql.queries.providers import ( BackupProvider as BackupProviderEnum, diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index 87e0aa6..bda7f09 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -1,3 +1,6 @@ +""" +Module for storing backup related data in redis. +""" from typing import List, Optional from datetime import datetime @@ -10,10 +13,6 @@ from selfprivacy_api.utils.redis_model_storage import ( hash_as_model, ) - -from selfprivacy_api.services.service import Service -from selfprivacy_api.services import get_service_by_id - from selfprivacy_api.backup.providers.provider import AbstractBackupProvider from selfprivacy_api.backup.providers import get_kind @@ -32,8 +31,10 @@ redis = RedisPool().get_connection() class Storage: + """Static class for storing backup related data in redis""" @staticmethod - def reset(): + def reset() -> None: + """Deletes all backup related data from redis""" redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) @@ -48,20 +49,22 @@ class Storage: redis.delete(key) @staticmethod - def invalidate_snapshot_storage(): + def invalidate_snapshot_storage() -> None: + """Deletes all cached snapshots from redis""" for key in redis.keys(REDIS_SNAPSHOTS_PREFIX + "*"): redis.delete(key) @staticmethod - def __last_backup_key(service_id): + def __last_backup_key(service_id: str) -> str: return REDIS_LAST_BACKUP_PREFIX + service_id @staticmethod - def __snapshot_key(snapshot: Snapshot): + def __snapshot_key(snapshot: Snapshot) -> str: return REDIS_SNAPSHOTS_PREFIX + snapshot.id @staticmethod def get_last_backup_time(service_id: str) -> Optional[datetime]: + """Returns last backup time for a service or None if it was never backed up""" key = Storage.__last_backup_key(service_id) if not redis.exists(key): return None @@ -72,7 +75,8 @@ class Storage: return snapshot.created_at @staticmethod - def store_last_timestamp(service_id: str, snapshot: Snapshot): + def store_last_timestamp(service_id: str, snapshot: Snapshot) -> None: + """Stores last backup time for a service""" store_model_as_hash( redis, Storage.__last_backup_key(service_id), @@ -80,18 +84,21 @@ class Storage: ) @staticmethod - def cache_snapshot(snapshot: Snapshot): + def cache_snapshot(snapshot: Snapshot) -> None: + """Stores snapshot metadata in redis for caching purposes""" snapshot_key = Storage.__snapshot_key(snapshot) store_model_as_hash(redis, snapshot_key, snapshot) redis.expire(snapshot_key, REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS) @staticmethod - def delete_cached_snapshot(snapshot: Snapshot): + def delete_cached_snapshot(snapshot: Snapshot) -> None: + """Deletes snapshot metadata from redis""" snapshot_key = Storage.__snapshot_key(snapshot) redis.delete(snapshot_key) @staticmethod def get_cached_snapshot_by_id(snapshot_id: str) -> Optional[Snapshot]: + """Returns cached snapshot by id or None if it doesn't exist""" key = REDIS_SNAPSHOTS_PREFIX + snapshot_id if not redis.exists(key): return None @@ -99,12 +106,14 @@ class Storage: @staticmethod def get_cached_snapshots() -> List[Snapshot]: - keys = redis.keys(REDIS_SNAPSHOTS_PREFIX + "*") - result = [] + """Returns all cached snapshots stored in redis""" + keys: list[str] = redis.keys(REDIS_SNAPSHOTS_PREFIX + "*") # type: ignore + result: list[Snapshot] = [] for key in keys: snapshot = hash_as_model(redis, key, Snapshot) - result.append(snapshot) + if snapshot: + result.append(snapshot) return result @staticmethod @@ -112,18 +121,21 @@ class Storage: """None means autobackup is disabled""" if not redis.exists(REDIS_AUTOBACKUP_PERIOD_KEY): return None - return int(redis.get(REDIS_AUTOBACKUP_PERIOD_KEY)) + return int(redis.get(REDIS_AUTOBACKUP_PERIOD_KEY)) # type: ignore @staticmethod - def store_autobackup_period_minutes(minutes: int): + def store_autobackup_period_minutes(minutes: int) -> None: + """Set the new autobackup period in minutes""" redis.set(REDIS_AUTOBACKUP_PERIOD_KEY, minutes) @staticmethod - def delete_backup_period(): + def delete_backup_period() -> None: + """Set the autobackup period to none, effectively disabling autobackup""" redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) @staticmethod - def store_provider(provider: AbstractBackupProvider): + def store_provider(provider: AbstractBackupProvider) -> None: + """Stores backup stroage provider auth data in redis""" store_model_as_hash( redis, REDIS_PROVIDER_KEY, @@ -138,6 +150,7 @@ class Storage: @staticmethod def load_provider() -> Optional[BackupProviderModel]: + """Loads backup storage provider auth data from redis""" provider_model = hash_as_model( redis, REDIS_PROVIDER_KEY, @@ -147,10 +160,12 @@ class Storage: @staticmethod def has_init_mark() -> bool: + """Returns True if the repository was initialized""" if redis.exists(REDIS_INITTED_CACHE_PREFIX): return True return False @staticmethod def mark_as_init(): + """Marks the repository as initialized""" redis.set(REDIS_INITTED_CACHE_PREFIX, 1) diff --git a/selfprivacy_api/backup/tasks.py b/selfprivacy_api/backup/tasks.py index ac8f2e2..0f73178 100644 --- a/selfprivacy_api/backup/tasks.py +++ b/selfprivacy_api/backup/tasks.py @@ -1,21 +1,24 @@ +""" +The tasks module contains the worker tasks that are used to back up and restore +""" from datetime import datetime from selfprivacy_api.graphql.common_types.backup import RestoreStrategy from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.utils.huey import huey -from selfprivacy_api.services import get_service_by_id from selfprivacy_api.services.service import Service from selfprivacy_api.backup import Backups -from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job -def validate_datetime(dt: datetime): - # dt = datetime.now(timezone.utc) +def validate_datetime(dt: datetime) -> bool: + """ + Validates that the datetime passed in is timezone-aware. + """ if dt.timetz is None: raise ValueError( """ - huey passed in the timezone-unaware time! + huey passed in the timezone-unaware time! Post it in support chat or maybe try uncommenting a line above """ ) @@ -25,6 +28,9 @@ def validate_datetime(dt: datetime): # huey tasks need to return something @huey.task() def start_backup(service: Service) -> bool: + """ + The worker task that starts the backup process. + """ Backups.back_up(service) return True @@ -34,12 +40,18 @@ def restore_snapshot( snapshot: Snapshot, strategy: RestoreStrategy = RestoreStrategy.DOWNLOAD_VERIFY_OVERWRITE, ) -> bool: + """ + The worker task that starts the restore process. + """ Backups.restore_snapshot(snapshot, strategy) return True @huey.periodic_task(validate_datetime=validate_datetime) def automatic_backup(): + """ + The worker periodic task that starts the automatic backup process. + """ time = datetime.now() for service in Backups.services_to_back_up(time): start_backup(service) diff --git a/selfprivacy_api/graphql/mutations/backup_mutations.py b/selfprivacy_api/graphql/mutations/backup_mutations.py index 5c8163c..b92af4a 100644 --- a/selfprivacy_api/graphql/mutations/backup_mutations.py +++ b/selfprivacy_api/graphql/mutations/backup_mutations.py @@ -1,7 +1,5 @@ -import datetime import typing import strawberry -from strawberry.types import Info from selfprivacy_api.graphql import IsAuthenticated from selfprivacy_api.graphql.mutations.mutation_interface import ( @@ -16,7 +14,7 @@ from selfprivacy_api.graphql.common_types.jobs import job_to_api_job from selfprivacy_api.graphql.common_types.backup import RestoreStrategy from selfprivacy_api.backup import Backups -from selfprivacy_api.services import get_all_services, get_service_by_id +from selfprivacy_api.services import get_service_by_id from selfprivacy_api.backup.tasks import start_backup, restore_snapshot from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job @@ -142,11 +140,11 @@ class BackupMutations: try: job = add_restore_job(snap) - except ValueError as e: + except ValueError as error: return GenericJobMutationReturn( success=False, code=400, - message=str(e), + message=str(error), job=None, ) diff --git a/selfprivacy_api/graphql/queries/backup.py b/selfprivacy_api/graphql/queries/backup.py index 322dab0..6535a88 100644 --- a/selfprivacy_api/graphql/queries/backup.py +++ b/selfprivacy_api/graphql/queries/backup.py @@ -64,6 +64,8 @@ class Backup: status=ServiceStatusEnum.OFF, url=None, dns_records=None, + can_be_backed_up=False, + backup_description="", ) else: service = service_to_graphql_service(service) diff --git a/selfprivacy_api/jobs/__init__.py b/selfprivacy_api/jobs/__init__.py index 2551237..ea1e15e 100644 --- a/selfprivacy_api/jobs/__init__.py +++ b/selfprivacy_api/jobs/__init__.py @@ -125,57 +125,57 @@ class Jobs: return False @staticmethod - def reset_logs(): + def reset_logs() -> None: redis = RedisPool().get_connection() for key in redis.keys(STATUS_LOGS_PREFIX + "*"): redis.delete(key) @staticmethod - def log_status_update(job: Job, status: JobStatus): + def log_status_update(job: Job, status: JobStatus) -> None: redis = RedisPool().get_connection() key = _status_log_key_from_uuid(job.uid) redis.lpush(key, status.value) redis.expire(key, 10) @staticmethod - def log_progress_update(job: Job, progress: int): + def log_progress_update(job: Job, progress: int) -> None: redis = RedisPool().get_connection() key = _progress_log_key_from_uuid(job.uid) redis.lpush(key, progress) redis.expire(key, 10) @staticmethod - def status_updates(job: Job) -> typing.List[JobStatus]: - result = [] + def status_updates(job: Job) -> list[JobStatus]: + result: list[JobStatus] = [] redis = RedisPool().get_connection() key = _status_log_key_from_uuid(job.uid) if not redis.exists(key): return [] - status_strings = redis.lrange(key, 0, -1) + status_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore for status in status_strings: try: result.append(JobStatus[status]) - except KeyError as e: - raise ValueError("impossible job status: " + status) from e + except KeyError as error: + raise ValueError("impossible job status: " + status) from error return result @staticmethod - def progress_updates(job: Job) -> typing.List[int]: - result = [] + def progress_updates(job: Job) -> list[int]: + result: list[int] = [] redis = RedisPool().get_connection() key = _progress_log_key_from_uuid(job.uid) if not redis.exists(key): return [] - progress_strings = redis.lrange(key, 0, -1) + progress_strings: list[str] = redis.lrange(key, 0, -1) # type: ignore for progress in progress_strings: try: result.append(int(progress)) - except KeyError as e: - raise ValueError("impossible job progress: " + progress) from e + except KeyError as error: + raise ValueError("impossible job progress: " + progress) from error return result @staticmethod @@ -257,19 +257,19 @@ class Jobs: return False -def _redis_key_from_uuid(uuid_string): +def _redis_key_from_uuid(uuid_string) -> str: return "jobs:" + str(uuid_string) -def _status_log_key_from_uuid(uuid_string): +def _status_log_key_from_uuid(uuid_string) -> str: return STATUS_LOGS_PREFIX + str(uuid_string) -def _progress_log_key_from_uuid(uuid_string): +def _progress_log_key_from_uuid(uuid_string) -> str: return PROGRESS_LOGS_PREFIX + str(uuid_string) -def _store_job_as_hash(redis, redis_key, model): +def _store_job_as_hash(redis, redis_key, model) -> None: for key, value in model.dict().items(): if isinstance(value, uuid.UUID): value = str(value) @@ -280,7 +280,7 @@ def _store_job_as_hash(redis, redis_key, model): redis.hset(redis_key, key, str(value)) -def _job_from_hash(redis, redis_key): +def _job_from_hash(redis, redis_key) -> typing.Optional[Job]: if redis.exists(redis_key): job_dict = redis.hgetall(redis_key) for date in [ diff --git a/selfprivacy_api/repositories/tokens/redis_tokens_repository.py b/selfprivacy_api/repositories/tokens/redis_tokens_repository.py index 8baa16d..80825bc 100644 --- a/selfprivacy_api/repositories/tokens/redis_tokens_repository.py +++ b/selfprivacy_api/repositories/tokens/redis_tokens_repository.py @@ -1,7 +1,7 @@ """ Token repository using Redis as backend. """ -from typing import Optional +from typing import Any, Optional from datetime import datetime from hashlib import md5 @@ -29,15 +29,15 @@ class RedisTokensRepository(AbstractTokensRepository): @staticmethod def token_key_for_device(device_name: str): - hash = md5() - hash.update(bytes(device_name, "utf-8")) - digest = hash.hexdigest() + md5_hash = md5() + md5_hash.update(bytes(device_name, "utf-8")) + digest = md5_hash.hexdigest() return TOKENS_PREFIX + digest def get_tokens(self) -> list[Token]: """Get the tokens""" redis = self.connection - token_keys = redis.keys(TOKENS_PREFIX + "*") + token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*") # type: ignore tokens = [] for key in token_keys: token = self._token_from_hash(key) @@ -45,10 +45,10 @@ class RedisTokensRepository(AbstractTokensRepository): tokens.append(token) return tokens - def _discover_token_key(self, input_token: Token) -> str: + def _discover_token_key(self, input_token: Token) -> Optional[str]: """brute-force searching for tokens, for robust deletion""" redis = self.connection - token_keys = redis.keys(TOKENS_PREFIX + "*") + token_keys: list[str] = redis.keys(TOKENS_PREFIX + "*") # type: ignore for key in token_keys: token = self._token_from_hash(key) if token == input_token: @@ -120,26 +120,26 @@ class RedisTokensRepository(AbstractTokensRepository): return self._new_device_key_from_hash(NEW_DEVICE_KEY_REDIS_KEY) @staticmethod - def _is_date_key(key: str): + def _is_date_key(key: str) -> bool: return key in [ "created_at", "expires_at", ] @staticmethod - def _prepare_model_dict(d: dict): - date_keys = [key for key in d.keys() if RedisTokensRepository._is_date_key(key)] + def _prepare_model_dict(model_dict: dict[str, Any]) -> None: + date_keys = [key for key in model_dict.keys() if RedisTokensRepository._is_date_key(key)] for date in date_keys: - if d[date] != "None": - d[date] = datetime.fromisoformat(d[date]) - for key in d.keys(): - if d[key] == "None": - d[key] = None + if model_dict[date] != "None": + model_dict[date] = datetime.fromisoformat(model_dict[date]) + for key in model_dict.keys(): + if model_dict[key] == "None": + model_dict[key] = None - def _model_dict_from_hash(self, redis_key: str) -> Optional[dict]: + def _model_dict_from_hash(self, redis_key: str) -> Optional[dict[str, Any]]: redis = self.connection if redis.exists(redis_key): - token_dict = redis.hgetall(redis_key) + token_dict: dict[str, Any] = redis.hgetall(redis_key) # type: ignore RedisTokensRepository._prepare_model_dict(token_dict) return token_dict return None diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py index 2f2cf21..4bd6eda 100644 --- a/selfprivacy_api/utils/redis_pool.py +++ b/selfprivacy_api/utils/redis_pool.py @@ -1,9 +1,9 @@ """ Redis pool module for selfprivacy_api """ +from os import environ import redis from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass -from os import environ REDIS_SOCKET = "/run/redis-sp-api/redis.sock" @@ -14,7 +14,7 @@ class RedisPool(metaclass=SingletonMetaclass): """ def __init__(self): - if "USE_REDIS_PORT" in environ.keys(): + if "USE_REDIS_PORT" in environ: self._pool = redis.ConnectionPool( host="127.0.0.1", port=int(environ["USE_REDIS_PORT"]),