From 0c61c1abb59eed6d92fba5e396ed3e3e6360fc33 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:36:11 +0300 Subject: [PATCH 1/6] Migrations mechanism and fix to the nixos-config branch --- selfprivacy_api/app.py | 5 ++- selfprivacy_api/migrations/__init__.py | 24 +++++++++++ .../migrations/fix_nixos_config_branch.py | 41 +++++++++++++++++++ selfprivacy_api/migrations/migration.py | 26 ++++++++++++ selfprivacy_api/resources/common.py | 2 +- 5 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 selfprivacy_api/migrations/__init__.py create mode 100644 selfprivacy_api/migrations/fix_nixos_config_branch.py create mode 100644 selfprivacy_api/migrations/migration.py diff --git a/selfprivacy_api/app.py b/selfprivacy_api/app.py index fc04aeb..3b575db 100644 --- a/selfprivacy_api/app.py +++ b/selfprivacy_api/app.py @@ -16,6 +16,8 @@ from selfprivacy_api.resources.services import services as api_services from selfprivacy_api.restic_controller.tasks import huey, init_restic +from selfprivacy_api.migrations import run_migrations + swagger_blueprint = get_swaggerui_blueprint( "/api/docs", "/api/swagger.json", config={"app_name": "SelfPrivacy API"} ) @@ -59,7 +61,7 @@ def create_app(test_config=None): def spec(): if app.config["ENABLE_SWAGGER"] == "1": swag = swagger(app) - swag["info"]["version"] = "1.1.0" + swag["info"]["version"] = "1.1.1" swag["info"]["title"] = "SelfPrivacy API" swag["info"]["description"] = "SelfPrivacy API" swag["securityDefinitions"] = { @@ -83,6 +85,7 @@ def create_app(test_config=None): if __name__ == "__main__": monkey.patch_all() created_app = create_app() + run_migrations() huey.start() init_restic() created_app.run(port=5050, debug=False) diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py new file mode 100644 index 0000000..6a7723c --- /dev/null +++ b/selfprivacy_api/migrations/__init__.py @@ -0,0 +1,24 @@ +from selfprivacy_api.utils import ReadUserData +from selfprivacy_api.migrations.fix_nixos_config_branch import FixNixosConfigBranch + +migrations = [ + FixNixosConfigBranch() +] + +def run_migrations(): + """ + Go over all migrations. If they are not skipped in userdata file, run them + if the migration needed. + """ + with ReadUserData() as data: + if "api" not in data: + skipped_migrations = [] + elif "skippedMigrations" not in data["api"]: + skipped_migrations = [] + else: + skipped_migrations = data["api"].get("skippedMigrations", []) + + for migration in migrations: + if migration.get_migration_name() not in skipped_migrations: + if migration.is_migration_needed(): + migration.migrate() diff --git a/selfprivacy_api/migrations/fix_nixos_config_branch.py b/selfprivacy_api/migrations/fix_nixos_config_branch.py new file mode 100644 index 0000000..af36c67 --- /dev/null +++ b/selfprivacy_api/migrations/fix_nixos_config_branch.py @@ -0,0 +1,41 @@ +import os +import subprocess + +from selfprivacy_api.migrations.migration import Migration + +class FixNixosConfigBranch(Migration): + def get_migration_name(self): + return "fix_nixos_config_branch" + + def get_migration_description(self): + return """Mobile SelfPrivacy app introduced a bug in version 0.4.0. + New servers were initialized with a rolling-testing nixos config branch. + This was fixed in app version 0.4.2, but existing servers were not updated. + This migration fixes this by changing the nixos config branch to master. + """ + + def is_migration_needed(self): + """Check the current branch of /etc/nixos and return True if it is rolling-testing""" + nixos_config_branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True) + if nixos_config_branch.decode("utf-8").strip() == "rolling-testing": + return True + else: + return False + + + def migrate(self): + """Affected server pulled the config with the --single-branch flag. + Git config remote.origin.fetch has to be changed, so all branches will be fetched. + Then, fetch all branches, pull and switch to master branch. + """ + print("Fixing Nixos config branch") + current_working_directory = os.getcwd() + os.chdir("/etc/nixos") + + subprocess.check_output(["git", "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"]) + subprocess.check_output(["git", "fetch", "--all"]) + subprocess.check_output(["git", "pull"]) + subprocess.check_output(["git", "checkout", "master"]) + + os.chdir(current_working_directory) + print("Done") \ No newline at end of file diff --git a/selfprivacy_api/migrations/migration.py b/selfprivacy_api/migrations/migration.py new file mode 100644 index 0000000..e5a37db --- /dev/null +++ b/selfprivacy_api/migrations/migration.py @@ -0,0 +1,26 @@ +from abc import ABC, abstractmethod + +""" +Abstract Migration class +This class is used to define the structure of a migration +Migration has a function is_migration_needed() that returns True or False +Migration has a function migrate() that does the migration +Migration has a function get_migration_name() that returns the migration name +Migration has a function get_migration_description() that returns the migration description +""" +class Migration(ABC): + @abstractmethod + def get_migration_name(self): + pass + + @abstractmethod + def get_migration_description(self): + pass + + @abstractmethod + def is_migration_needed(self): + pass + + @abstractmethod + def migrate(self): + pass diff --git a/selfprivacy_api/resources/common.py b/selfprivacy_api/resources/common.py index ba7412c..d771372 100644 --- a/selfprivacy_api/resources/common.py +++ b/selfprivacy_api/resources/common.py @@ -24,4 +24,4 @@ class ApiVersion(Resource): 401: description: Unauthorized """ - return {"version": "1.1.0"} + return {"version": "1.1.1"} From 650032bdab5e7fa4556aaed5575a008c9d1db65d Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:41:25 +0300 Subject: [PATCH 2/6] Formatting --- selfprivacy_api/migrations/__init__.py | 5 ++--- .../migrations/fix_nixos_config_branch.py | 17 +++++++++++++---- selfprivacy_api/migrations/migration.py | 2 ++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py index 6a7723c..61097a7 100644 --- a/selfprivacy_api/migrations/__init__.py +++ b/selfprivacy_api/migrations/__init__.py @@ -1,9 +1,8 @@ from selfprivacy_api.utils import ReadUserData from selfprivacy_api.migrations.fix_nixos_config_branch import FixNixosConfigBranch -migrations = [ - FixNixosConfigBranch() -] +migrations = [FixNixosConfigBranch()] + def run_migrations(): """ diff --git a/selfprivacy_api/migrations/fix_nixos_config_branch.py b/selfprivacy_api/migrations/fix_nixos_config_branch.py index af36c67..26f4ff7 100644 --- a/selfprivacy_api/migrations/fix_nixos_config_branch.py +++ b/selfprivacy_api/migrations/fix_nixos_config_branch.py @@ -3,6 +3,7 @@ import subprocess from selfprivacy_api.migrations.migration import Migration + class FixNixosConfigBranch(Migration): def get_migration_name(self): return "fix_nixos_config_branch" @@ -16,12 +17,13 @@ class FixNixosConfigBranch(Migration): def is_migration_needed(self): """Check the current branch of /etc/nixos and return True if it is rolling-testing""" - nixos_config_branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True) + nixos_config_branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True + ) if nixos_config_branch.decode("utf-8").strip() == "rolling-testing": return True else: return False - def migrate(self): """Affected server pulled the config with the --single-branch flag. @@ -32,10 +34,17 @@ class FixNixosConfigBranch(Migration): current_working_directory = os.getcwd() os.chdir("/etc/nixos") - subprocess.check_output(["git", "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"]) + subprocess.check_output( + [ + "git", + "config", + "remote.origin.fetch", + "+refs/heads/*:refs/remotes/origin/*", + ] + ) subprocess.check_output(["git", "fetch", "--all"]) subprocess.check_output(["git", "pull"]) subprocess.check_output(["git", "checkout", "master"]) os.chdir(current_working_directory) - print("Done") \ No newline at end of file + print("Done") diff --git a/selfprivacy_api/migrations/migration.py b/selfprivacy_api/migrations/migration.py index e5a37db..b8ae261 100644 --- a/selfprivacy_api/migrations/migration.py +++ b/selfprivacy_api/migrations/migration.py @@ -8,6 +8,8 @@ Migration has a function migrate() that does the migration Migration has a function get_migration_name() that returns the migration name Migration has a function get_migration_description() that returns the migration description """ + + class Migration(ABC): @abstractmethod def get_migration_name(self): From 07cf926e79b6de88dc43ce953708274332841fa2 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:52:15 +0300 Subject: [PATCH 3/6] Fix migration crashes. --- selfprivacy_api/migrations/__init__.py | 7 ++- .../migrations/fix_nixos_config_branch.py | 51 +++++++++++-------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py index 61097a7..67c9107 100644 --- a/selfprivacy_api/migrations/__init__.py +++ b/selfprivacy_api/migrations/__init__.py @@ -20,4 +20,9 @@ def run_migrations(): for migration in migrations: if migration.get_migration_name() not in skipped_migrations: if migration.is_migration_needed(): - migration.migrate() + try: + migration.migrate() + except Exception as e: + print(f"Error while migrating {migration.get_migration_name()}") + print(e) + print("Skipping this migration") diff --git a/selfprivacy_api/migrations/fix_nixos_config_branch.py b/selfprivacy_api/migrations/fix_nixos_config_branch.py index 26f4ff7..450a46b 100644 --- a/selfprivacy_api/migrations/fix_nixos_config_branch.py +++ b/selfprivacy_api/migrations/fix_nixos_config_branch.py @@ -17,12 +17,19 @@ class FixNixosConfigBranch(Migration): def is_migration_needed(self): """Check the current branch of /etc/nixos and return True if it is rolling-testing""" - nixos_config_branch = subprocess.check_output( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True - ) - if nixos_config_branch.decode("utf-8").strip() == "rolling-testing": - return True - else: + current_working_directory = os.getcwd() + try: + os.chdir("/etc/nixos") + nixos_config_branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True + ) + os.chdir(current_working_directory) + if nixos_config_branch.decode("utf-8").strip() == "rolling-testing": + return True + else: + return False + except subprocess.CalledProcessError: + os.chdir(current_working_directory) return False def migrate(self): @@ -32,19 +39,23 @@ class FixNixosConfigBranch(Migration): """ print("Fixing Nixos config branch") current_working_directory = os.getcwd() - os.chdir("/etc/nixos") + try: + os.chdir("/etc/nixos") - subprocess.check_output( - [ - "git", - "config", - "remote.origin.fetch", - "+refs/heads/*:refs/remotes/origin/*", - ] - ) - subprocess.check_output(["git", "fetch", "--all"]) - subprocess.check_output(["git", "pull"]) - subprocess.check_output(["git", "checkout", "master"]) + subprocess.check_output( + [ + "git", + "config", + "remote.origin.fetch", + "+refs/heads/*:refs/remotes/origin/*", + ] + ) + subprocess.check_output(["git", "fetch", "--all"]) + subprocess.check_output(["git", "pull"]) + subprocess.check_output(["git", "checkout", "master"]) + os.chdir(current_working_directory) + print("Done") + except subprocess.CalledProcessError: + os.chdir(current_working_directory) + print("Error") - os.chdir(current_working_directory) - print("Done") From 355fc68232ff6a6b530969b33b94b4b1015f12e2 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:52:43 +0300 Subject: [PATCH 4/6] Fix newline --- selfprivacy_api/migrations/fix_nixos_config_branch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfprivacy_api/migrations/fix_nixos_config_branch.py b/selfprivacy_api/migrations/fix_nixos_config_branch.py index 450a46b..cb1907d 100644 --- a/selfprivacy_api/migrations/fix_nixos_config_branch.py +++ b/selfprivacy_api/migrations/fix_nixos_config_branch.py @@ -58,4 +58,3 @@ class FixNixosConfigBranch(Migration): except subprocess.CalledProcessError: os.chdir(current_working_directory) print("Error") - From 6c0af38e27409e1091b7dca37e69caa9fe60686f Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:54:57 +0300 Subject: [PATCH 5/6] Move is_migration_needed call under try block too --- selfprivacy_api/migrations/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py index 67c9107..5fb4c72 100644 --- a/selfprivacy_api/migrations/__init__.py +++ b/selfprivacy_api/migrations/__init__.py @@ -19,10 +19,10 @@ def run_migrations(): for migration in migrations: if migration.get_migration_name() not in skipped_migrations: - if migration.is_migration_needed(): - try: + try: + if migration.is_migration_needed(): migration.migrate() - except Exception as e: - print(f"Error while migrating {migration.get_migration_name()}") - print(e) - print("Skipping this migration") + except Exception as e: + print(f"Error while migrating {migration.get_migration_name()}") + print(e) + print("Skipping this migration") From aa76f87828a98d892f68211193d63321f23937fc Mon Sep 17 00:00:00 2001 From: Inex Code Date: Tue, 11 Jan 2022 08:57:14 +0300 Subject: [PATCH 6/6] Allow user to disable all migrations entirely --- selfprivacy_api/migrations/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py index 5fb4c72..32c467f 100644 --- a/selfprivacy_api/migrations/__init__.py +++ b/selfprivacy_api/migrations/__init__.py @@ -17,6 +17,9 @@ def run_migrations(): else: skipped_migrations = data["api"].get("skippedMigrations", []) + if "DISABLE_ALL" in skipped_migrations: + return + for migration in migrations: if migration.get_migration_name() not in skipped_migrations: try: