From 83592b7bf4173c9d692b1fe1ae087195c7a06b6f Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Fri, 19 Jan 2024 14:06:07 +0000
Subject: [PATCH 1/8] feature(huey): use RedisHuey

---
 selfprivacy_api/task_registry.py    |  1 +
 selfprivacy_api/utils/huey.py       | 23 ++++++++++------
 selfprivacy_api/utils/redis_pool.py | 29 +++++++++++---------
 tests/conftest.py                   | 15 +++--------
 tests/test_huey.py                  | 41 +++++++++++++++++++++++++++++
 5 files changed, 77 insertions(+), 32 deletions(-)
 create mode 100644 tests/test_huey.py

diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py
index a492e23..b57bfbe 100644
--- a/selfprivacy_api/task_registry.py
+++ b/selfprivacy_api/task_registry.py
@@ -3,3 +3,4 @@ from selfprivacy_api.jobs.test import test_job
 from selfprivacy_api.backup.tasks import *
 from selfprivacy_api.services.tasks import move_service
 from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
+from tests.test_huey import sum
diff --git a/selfprivacy_api/utils/huey.py b/selfprivacy_api/utils/huey.py
index 8e09446..2276d37 100644
--- a/selfprivacy_api/utils/huey.py
+++ b/selfprivacy_api/utils/huey.py
@@ -1,16 +1,23 @@
 """MiniHuey singleton."""
 import os
-from huey import SqliteHuey
+from os import environ
+from huey import RedisHuey
 
-HUEY_DATABASE = "/etc/selfprivacy/tasks.db"
+from selfprivacy_api.utils.redis_pool import RedisPool
+
+HUEY_DATABASE_NUMBER = 10
+
+def immediate() -> bool:
+    if environ.get("HUEY_QUEUES_FOR_TESTS"):
+        return False
+    if environ.get("TEST_MODE"):
+        return True
+    return False
 
 # Singleton instance containing the huey database.
-
-test_mode = os.environ.get("TEST_MODE")
-
-huey = SqliteHuey(
+huey = RedisHuey(
     "selfprivacy-api",
-    filename=HUEY_DATABASE if not test_mode else None,
-    immediate=test_mode == "true",
+    url=RedisPool.connection_url(dbnumber=HUEY_DATABASE_NUMBER),
+    immediate=immediate(),
     utc=True,
 )
diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 4bd6eda..d9076d2 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -14,20 +14,25 @@ class RedisPool(metaclass=SingletonMetaclass):
     """
 
     def __init__(self):
-        if "USE_REDIS_PORT" in environ:
-            self._pool = redis.ConnectionPool(
-                host="127.0.0.1",
-                port=int(environ["USE_REDIS_PORT"]),
-                decode_responses=True,
-            )
-
-        else:
-            self._pool = redis.ConnectionPool.from_url(
-                f"unix://{REDIS_SOCKET}",
-                decode_responses=True,
-            )
+        self._pool = redis.ConnectionPool.from_url(
+            RedisPool.connection_url(dbnumber=0),
+            decode_responses=True,
+        )
         self._pubsub_connection = self.get_connection()
 
+    @staticmethod
+    def connection_url(dbnumber: int) -> str:
+        """
+        redis://[[username]:[password]]@localhost:6379/0
+        unix://[username@]/path/to/socket.sock?db=0[&password=password]
+        """
+
+        if "USE_REDIS_PORT" in environ:
+            port = int(environ["USE_REDIS_PORT"])
+            return f"redis://@127.0.0.1:{port}/{dbnumber}"
+        else:
+            return f"unix://{REDIS_SOCKET}?db={dbnumber}"
+
     def get_connection(self):
         """
         Get a connection from the pool.
diff --git a/tests/conftest.py b/tests/conftest.py
index dceac72..f1c6e89 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -99,23 +99,14 @@ def generic_userdata(mocker, tmpdir):
 
 
 @pytest.fixture
-def huey_database(mocker, shared_datadir):
-    """Mock huey database."""
-    mock = mocker.patch(
-        "selfprivacy_api.utils.huey.HUEY_DATABASE", shared_datadir / "huey.db"
-    )
-    return mock
-
-
-@pytest.fixture
-def client(huey_database, redis_repo_with_tokens):
+def client(redis_repo_with_tokens):
     from selfprivacy_api.app import app
 
     return TestClient(app)
 
 
 @pytest.fixture
-def authorized_client(huey_database, redis_repo_with_tokens):
+def authorized_client(redis_repo_with_tokens):
     """Authorized test client fixture."""
     from selfprivacy_api.app import app
 
@@ -127,7 +118,7 @@ def authorized_client(huey_database, redis_repo_with_tokens):
 
 
 @pytest.fixture
-def wrong_auth_client(huey_database, redis_repo_with_tokens):
+def wrong_auth_client(redis_repo_with_tokens):
     """Wrong token test client fixture."""
     from selfprivacy_api.app import app
 
diff --git a/tests/test_huey.py b/tests/test_huey.py
new file mode 100644
index 0000000..49103f2
--- /dev/null
+++ b/tests/test_huey.py
@@ -0,0 +1,41 @@
+import pytest
+
+from subprocess import Popen
+from os import environ
+
+
+# from selfprivacy_api.backup.util import output_yielder
+from selfprivacy_api.utils.huey import huey
+
+
+@huey.task()
+def sum(a: int, b: int) -> int:
+    return a + b
+
+
+@pytest.fixture()
+def huey_queues():
+    """
+    Full, not-immediate, queued huey, with consumer starting and stopping.
+    IMPORTANT: Assumes tests are run from the project directory.
+    The above is needed by consumer to find our huey setup.
+    """
+    old_immediate = huey.immediate
+
+    environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
+    command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
+    huey.immediate = False
+    assert huey.immediate is False
+    consumer_handle = Popen(command)
+
+    yield huey
+
+    consumer_handle.terminate()
+    del environ["HUEY_QUEUES_FOR_TESTS"]
+    huey.immediate = old_immediate
+    assert huey.immediate == old_immediate
+
+
+def test_huey(huey_queues):
+    result = sum(2, 5)
+    assert result(blocking=True) == 7

From 2b19633cbde057dd37ef51d19b52f20751941531 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Fri, 19 Jan 2024 15:03:00 +0000
Subject: [PATCH 2/8] test(huey): break out preparing the environment vars

I did it for testing redis socketing too, but I guess this will wait for
another time. Somehow it worked even without an actual redis socket and it was
creepy. Idk yet how one can best make redis to make sockets at arbitrary
temporary dirs without starting another redis.
---
 tests/test_huey.py | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/tests/test_huey.py b/tests/test_huey.py
index 49103f2..fc711ac 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -1,11 +1,9 @@
 import pytest
 
 from subprocess import Popen
-from os import environ
+from os import environ, path
 
-
-# from selfprivacy_api.backup.util import output_yielder
-from selfprivacy_api.utils.huey import huey
+from selfprivacy_api.utils.huey import huey, immediate
 
 
 @huey.task()
@@ -14,28 +12,37 @@ def sum(a: int, b: int) -> int:
 
 
 @pytest.fixture()
-def huey_queues():
-    """
-    Full, not-immediate, queued huey, with consumer starting and stopping.
-    IMPORTANT: Assumes tests are run from the project directory.
-    The above is needed by consumer to find our huey setup.
-    """
+def not_immediate():
     old_immediate = huey.immediate
-
     environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
-    command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
     huey.immediate = False
     assert huey.immediate is False
-    consumer_handle = Popen(command)
 
-    yield huey
+    yield
 
-    consumer_handle.terminate()
     del environ["HUEY_QUEUES_FOR_TESTS"]
     huey.immediate = old_immediate
     assert huey.immediate == old_immediate
 
 
+@pytest.fixture()
+def huey_queues(not_immediate):
+    """
+    Full, not-immediate, queued huey, with consumer starting and stopping.
+    IMPORTANT: Assumes tests are run from the project directory.
+    The above is needed by consumer to find our huey setup.
+    """
+    command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
+    consumer_handle = Popen(command)
+
+    yield huey
+
+    consumer_handle.terminate()
+
+
 def test_huey(huey_queues):
+    assert huey.immediate is False
+    assert immediate() is False
+
     result = sum(2, 5)
     assert result(blocking=True) == 7

From 9954737791c8999cfb6f3fc0f753c8c031bedf25 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Fri, 19 Jan 2024 15:33:01 +0000
Subject: [PATCH 3/8] use kill() instead of terminate in huey tests

---
 tests/test_huey.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_huey.py b/tests/test_huey.py
index fc711ac..6ff780a 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -37,7 +37,7 @@ def huey_queues(not_immediate):
 
     yield huey
 
-    consumer_handle.terminate()
+    consumer_handle.kill()
 
 
 def test_huey(huey_queues):

From fde461b4b96d405737455ff11ab281fa6aad28cb Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 24 Jan 2024 11:44:46 +0000
Subject: [PATCH 4/8] test(huey): test that redis socket connection works

---
 selfprivacy_api/utils/redis_pool.py |  11 ++-
 tests/test_huey.py                  | 120 +++++++++++++++++++++++++++-
 2 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index d9076d2..808c1ac 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -5,8 +5,6 @@ from os import environ
 import redis
 from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 
-REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
-
 
 class RedisPool(metaclass=SingletonMetaclass):
     """
@@ -31,7 +29,14 @@ class RedisPool(metaclass=SingletonMetaclass):
             port = int(environ["USE_REDIS_PORT"])
             return f"redis://@127.0.0.1:{port}/{dbnumber}"
         else:
-            return f"unix://{REDIS_SOCKET}?db={dbnumber}"
+            return f"unix://{RedisPool.redis_socket()}?db={dbnumber}"
+
+    @staticmethod
+    def redis_socket() -> str:
+        if "REDIS_SOCKET" in environ:
+            return environ["REDIS_SOCKET"]
+        else:
+            return "/run/redis-sp-api/redis.sock"
 
     def get_connection(self):
         """
diff --git a/tests/test_huey.py b/tests/test_huey.py
index 6ff780a..f68bef3 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -2,8 +2,12 @@ import pytest
 
 from subprocess import Popen
 from os import environ, path
+import redis
 
-from selfprivacy_api.utils.huey import huey, immediate
+from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
+from selfprivacy_api.backup.util import output_yielder
+from selfprivacy_api.utils.redis_pool import RedisPool
+from selfprivacy_api.utils.waitloop import wait_until_true
 
 
 @huey.task()
@@ -11,6 +15,70 @@ def sum(a: int, b: int) -> int:
     return a + b
 
 
+def reset_huey_storage():
+    huey.storage = huey.create_storage()
+
+
+def flush_huey_redis_forcefully():
+    url = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
+
+    pool = redis.ConnectionPool.from_url(url, decode_responses=True)
+    connection = redis.Redis(connection_pool=pool)
+    connection.flushdb()
+
+
+@pytest.fixture()
+def redis_socket(tmpdir):
+    # Does NOT overwrite already imported redis pools
+    # -> Not very useful for more involved tests
+    # DOES override imported huey partially, but tries to restore it back
+
+    socket_path = path.join(tmpdir, "redis.sock")
+    environ["REDIS_SOCKET"] = socket_path
+
+    old_port = None
+    if "USE_REDIS_PORT" in environ:
+        old_port = environ["USE_REDIS_PORT"]
+        del environ["USE_REDIS_PORT"]
+
+    assert "USE_REDIS_PORT" not in environ
+
+    old_huey_url = huey.storage_kwargs.get("url")
+    # Overriding url in the already imported singleton
+    huey.storage_kwargs["url"] = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
+    reset_huey_storage()
+
+    # Socket file will be created by redis
+    command = [
+        "redis-server",
+        "--unixsocket",
+        socket_path,
+        "--unixsocketperm",
+        "700",
+        "--port",
+        "0",
+    ]
+    redis_handle = Popen(command)
+
+    wait_until_true(lambda: path.exists(socket_path), timeout_sec=2)
+    flush_huey_redis_forcefully()
+
+    yield socket_path
+
+    # Socket file will be destroyed by redis
+    redis_handle.terminate()
+
+    if old_port:
+        environ["USE_REDIS_PORT"] = old_port
+    del environ["REDIS_SOCKET"]
+    if old_huey_url:
+        huey.storage_kwargs["url"] = old_huey_url
+    else:
+        del huey.storage_kwargs["url"]
+
+    reset_huey_storage()
+
+
 @pytest.fixture()
 def not_immediate():
     old_immediate = huey.immediate
@@ -32,6 +100,7 @@ def huey_queues(not_immediate):
     IMPORTANT: Assumes tests are run from the project directory.
     The above is needed by consumer to find our huey setup.
     """
+    flush_huey_redis_forcefully()
     command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
     consumer_handle = Popen(command)
 
@@ -40,9 +109,54 @@ def huey_queues(not_immediate):
     consumer_handle.kill()
 
 
-def test_huey(huey_queues):
+@pytest.fixture()
+def huey_queues_socket(not_immediate, redis_socket):
+    """
+    Same as above, but with socketed redis
+    """
+
+    flush_huey_redis_forcefully()
+    command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
+    consumer_handle = Popen(command)
+
+    assert path.exists(redis_socket)
+
+    yield redis_socket
+
+    consumer_handle.kill()
+
+
+def test_huey_over_redis(huey_queues):
     assert huey.immediate is False
     assert immediate() is False
 
     result = sum(2, 5)
-    assert result(blocking=True) == 7
+    assert result(blocking=True, timeout=2) == 7
+
+
+# we cannot have these two fixtures prepared at the same time to iterate through them
+def test_huey_over_redis_socket(huey_queues_socket):
+    assert huey.immediate is False
+    assert immediate() is False
+
+    assert "unix" in RedisPool.connection_url(HUEY_DATABASE_NUMBER)
+    try:
+        assert (
+            RedisPool.connection_url(HUEY_DATABASE_NUMBER)
+            in huey.storage_kwargs.values()
+        )
+    except AssertionError:
+        raise ValueError(
+            "our test-side huey does not connect over socket: ", huey.storage_kwargs
+        )
+
+    # for some reason this fails. We do not schedule tasks anywhere, but concerning.
+    # result = sum.schedule((2, 5), delay=10)
+    # try:
+    #     assert len(huey.scheduled()) == 1
+    # except AssertionError:
+    #     raise ValueError("have wrong amount of scheduled tasks", huey.scheduled())
+    # result.revoke()
+
+    result = sum(2, 5)
+    assert result(blocking=True, timeout=2) == 7

From 8e48a5ad5f720b4ccbc703d269aa42fdeeaf11f4 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 24 Jan 2024 12:19:25 +0000
Subject: [PATCH 5/8] test(huey): add a scheduling test (expected-fails for
 now)

---
 tests/test_huey.py | 57 +++++++++++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 24 deletions(-)

diff --git a/tests/test_huey.py b/tests/test_huey.py
index f68bef3..8b8e77b 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -5,10 +5,11 @@ from os import environ, path
 import redis
 
 from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
-from selfprivacy_api.backup.util import output_yielder
 from selfprivacy_api.utils.redis_pool import RedisPool
 from selfprivacy_api.utils.waitloop import wait_until_true
 
+from selfprivacy_api.backup.util import output_yielder
+
 
 @huey.task()
 def sum(a: int, b: int) -> int:
@@ -27,6 +28,25 @@ def flush_huey_redis_forcefully():
     connection.flushdb()
 
 
+def start_redis_socket(socket_path):
+    # Socket file will be created by redis
+    command = [
+        "redis-server",
+        "--unixsocket",
+        socket_path,
+        "--unixsocketperm",
+        "700",
+        "--port",
+        "0",
+    ]
+    redis_handle = Popen(command)
+
+    wait_until_true(lambda: path.exists(socket_path), timeout_sec=2)
+    flush_huey_redis_forcefully()
+
+    return redis_handle
+
+
 @pytest.fixture()
 def redis_socket(tmpdir):
     # Does NOT overwrite already imported redis pools
@@ -48,20 +68,7 @@ def redis_socket(tmpdir):
     huey.storage_kwargs["url"] = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
     reset_huey_storage()
 
-    # Socket file will be created by redis
-    command = [
-        "redis-server",
-        "--unixsocket",
-        socket_path,
-        "--unixsocketperm",
-        "700",
-        "--port",
-        "0",
-    ]
-    redis_handle = Popen(command)
-
-    wait_until_true(lambda: path.exists(socket_path), timeout_sec=2)
-    flush_huey_redis_forcefully()
+    redis_handle = start_redis_socket(socket_path)
 
     yield socket_path
 
@@ -71,11 +78,11 @@ def redis_socket(tmpdir):
     if old_port:
         environ["USE_REDIS_PORT"] = old_port
     del environ["REDIS_SOCKET"]
+
     if old_huey_url:
         huey.storage_kwargs["url"] = old_huey_url
     else:
         del huey.storage_kwargs["url"]
-
     reset_huey_storage()
 
 
@@ -150,13 +157,15 @@ def test_huey_over_redis_socket(huey_queues_socket):
             "our test-side huey does not connect over socket: ", huey.storage_kwargs
         )
 
-    # for some reason this fails. We do not schedule tasks anywhere, but concerning.
-    # result = sum.schedule((2, 5), delay=10)
-    # try:
-    #     assert len(huey.scheduled()) == 1
-    # except AssertionError:
-    #     raise ValueError("have wrong amount of scheduled tasks", huey.scheduled())
-    # result.revoke()
-
     result = sum(2, 5)
     assert result(blocking=True, timeout=2) == 7
+
+
+@pytest.mark.xfail(reason="cannot yet schedule with sockets for some reason")
+def test_huey_schedule(huey_queues_socket):
+    # we do not schedule tasks anywhere, but concerning.
+    result = sum.schedule((2, 5), delay=10)
+    try:
+        assert len(huey.scheduled()) == 1
+    except assertionerror:
+        raise valueerror("have wrong amount of scheduled tasks", huey.scheduled())

From baf784334997f039ec12c8763374966f63170237 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 24 Jan 2024 12:40:55 +0000
Subject: [PATCH 6/8] test(huey): only import test task if it is a test

---
 selfprivacy_api/task_registry.py | 10 ++++++++--
 selfprivacy_api/utils/huey.py    |  3 ++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py
index b57bfbe..6e1518d 100644
--- a/selfprivacy_api/task_registry.py
+++ b/selfprivacy_api/task_registry.py
@@ -1,6 +1,12 @@
+from os import environ
+
 from selfprivacy_api.utils.huey import huey
-from selfprivacy_api.jobs.test import test_job
+
 from selfprivacy_api.backup.tasks import *
 from selfprivacy_api.services.tasks import move_service
 from selfprivacy_api.jobs.upgrade_system import rebuild_system_task
-from tests.test_huey import sum
+
+from selfprivacy_api.jobs.test import test_job
+
+if environ.get("TEST_MODE"):
+    from tests.test_huey import sum
diff --git a/selfprivacy_api/utils/huey.py b/selfprivacy_api/utils/huey.py
index 2276d37..1a7a29d 100644
--- a/selfprivacy_api/utils/huey.py
+++ b/selfprivacy_api/utils/huey.py
@@ -1,5 +1,4 @@
 """MiniHuey singleton."""
-import os
 from os import environ
 from huey import RedisHuey
 
@@ -7,6 +6,7 @@ from selfprivacy_api.utils.redis_pool import RedisPool
 
 HUEY_DATABASE_NUMBER = 10
 
+
 def immediate() -> bool:
     if environ.get("HUEY_QUEUES_FOR_TESTS"):
         return False
@@ -14,6 +14,7 @@ def immediate() -> bool:
         return True
     return False
 
+
 # Singleton instance containing the huey database.
 huey = RedisHuey(
     "selfprivacy-api",

From 6f38b2309fa497fff81da656d5cbb6d47df62018 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 6 Mar 2024 15:08:04 +0000
Subject: [PATCH 7/8] fix(huey): adapt to new VM test environment

---
 flake.nix                           |   1 +
 selfprivacy_api/utils/redis_pool.py |  18 +---
 tests/test_huey.py                  | 153 +++++++++++-----------------
 3 files changed, 62 insertions(+), 110 deletions(-)

diff --git a/flake.nix b/flake.nix
index fee8e79..f8b81aa 100644
--- a/flake.nix
+++ b/flake.nix
@@ -27,6 +27,7 @@
             python-lsp-server
             pyflakes
             typer # for strawberry
+            types-redis # for mypy
           ] ++ strawberry-graphql.optional-dependencies.cli));
 
       vmtest-src-dir = "/root/source";
diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 808c1ac..3d35f01 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -1,10 +1,12 @@
 """
 Redis pool module for selfprivacy_api
 """
-from os import environ
 import redis
+
 from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 
+REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
+
 
 class RedisPool(metaclass=SingletonMetaclass):
     """
@@ -24,19 +26,7 @@ class RedisPool(metaclass=SingletonMetaclass):
         redis://[[username]:[password]]@localhost:6379/0
         unix://[username@]/path/to/socket.sock?db=0[&password=password]
         """
-
-        if "USE_REDIS_PORT" in environ:
-            port = int(environ["USE_REDIS_PORT"])
-            return f"redis://@127.0.0.1:{port}/{dbnumber}"
-        else:
-            return f"unix://{RedisPool.redis_socket()}?db={dbnumber}"
-
-    @staticmethod
-    def redis_socket() -> str:
-        if "REDIS_SOCKET" in environ:
-            return environ["REDIS_SOCKET"]
-        else:
-            return "/run/redis-sp-api/redis.sock"
+        return f"unix://{REDIS_SOCKET}?db={dbnumber}"
 
     def get_connection(self):
         """
diff --git a/tests/test_huey.py b/tests/test_huey.py
index 8b8e77b..b14be9a 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -1,14 +1,15 @@
 import pytest
-
-from subprocess import Popen
-from os import environ, path
 import redis
+from typing import List
+
+import subprocess
+from subprocess import Popen, check_output, TimeoutExpired
+from os import environ, path, set_blocking
+from io import BufferedReader
+from huey.exceptions import HueyException
 
 from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
-from selfprivacy_api.utils.redis_pool import RedisPool
-from selfprivacy_api.utils.waitloop import wait_until_true
-
-from selfprivacy_api.backup.util import output_yielder
+from selfprivacy_api.utils.redis_pool import RedisPool, REDIS_SOCKET
 
 
 @huey.task()
@@ -28,66 +29,28 @@ def flush_huey_redis_forcefully():
     connection.flushdb()
 
 
-def start_redis_socket(socket_path):
-    # Socket file will be created by redis
-    command = [
-        "redis-server",
-        "--unixsocket",
-        socket_path,
-        "--unixsocketperm",
-        "700",
-        "--port",
-        "0",
-    ]
-    redis_handle = Popen(command)
+# TODO: may be useful in other places too, move to utils/ tests common if using it somewhere
+def read_all_ready_output(stream: BufferedReader) -> str:
+    set_blocking(stream.fileno(), False)
+    output: List[bytes] = []
+    while True:
+        line = stream.readline()
+        raise ValueError(line)
+        if line == b"":
+            break
+        else:
+            output.append(line)
 
-    wait_until_true(lambda: path.exists(socket_path), timeout_sec=2)
-    flush_huey_redis_forcefully()
+    set_blocking(stream.fileno(), True)
 
-    return redis_handle
-
-
-@pytest.fixture()
-def redis_socket(tmpdir):
-    # Does NOT overwrite already imported redis pools
-    # -> Not very useful for more involved tests
-    # DOES override imported huey partially, but tries to restore it back
-
-    socket_path = path.join(tmpdir, "redis.sock")
-    environ["REDIS_SOCKET"] = socket_path
-
-    old_port = None
-    if "USE_REDIS_PORT" in environ:
-        old_port = environ["USE_REDIS_PORT"]
-        del environ["USE_REDIS_PORT"]
-
-    assert "USE_REDIS_PORT" not in environ
-
-    old_huey_url = huey.storage_kwargs.get("url")
-    # Overriding url in the already imported singleton
-    huey.storage_kwargs["url"] = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
-    reset_huey_storage()
-
-    redis_handle = start_redis_socket(socket_path)
-
-    yield socket_path
-
-    # Socket file will be destroyed by redis
-    redis_handle.terminate()
-
-    if old_port:
-        environ["USE_REDIS_PORT"] = old_port
-    del environ["REDIS_SOCKET"]
-
-    if old_huey_url:
-        huey.storage_kwargs["url"] = old_huey_url
-    else:
-        del huey.storage_kwargs["url"]
-    reset_huey_storage()
+    result = b"".join(output)
+    return result.decode("utf-8")
 
 
 @pytest.fixture()
 def not_immediate():
+    assert environ["TEST_MODE"] == "true"
+
     old_immediate = huey.immediate
     environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
     huey.immediate = False
@@ -101,48 +64,32 @@ def not_immediate():
 
 
 @pytest.fixture()
-def huey_queues(not_immediate):
-    """
-    Full, not-immediate, queued huey, with consumer starting and stopping.
-    IMPORTANT: Assumes tests are run from the project directory.
-    The above is needed by consumer to find our huey setup.
-    """
-    flush_huey_redis_forcefully()
-    command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
-    consumer_handle = Popen(command)
-
-    yield huey
-
-    consumer_handle.kill()
-
-
-@pytest.fixture()
-def huey_queues_socket(not_immediate, redis_socket):
+def huey_socket_consumer(not_immediate):
     """
     Same as above, but with socketed redis
     """
 
     flush_huey_redis_forcefully()
     command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
-    consumer_handle = Popen(command)
 
-    assert path.exists(redis_socket)
+    # First assert that consumer does not fail by itself
+    # Idk yet how to do it more elegantly
+    try:
+        check_output(command, timeout=2)
+    except TimeoutExpired:
+        pass
 
-    yield redis_socket
+    # Then open it for real
+    consumer_handle = Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    assert path.exists(REDIS_SOCKET)
+
+    yield consumer_handle
 
     consumer_handle.kill()
 
 
-def test_huey_over_redis(huey_queues):
-    assert huey.immediate is False
-    assert immediate() is False
-
-    result = sum(2, 5)
-    assert result(blocking=True, timeout=2) == 7
-
-
-# we cannot have these two fixtures prepared at the same time to iterate through them
-def test_huey_over_redis_socket(huey_queues_socket):
+def test_huey_over_redis_socket(huey_socket_consumer):
     assert huey.immediate is False
     assert immediate() is False
 
@@ -158,14 +105,28 @@ def test_huey_over_redis_socket(huey_queues_socket):
         )
 
     result = sum(2, 5)
-    assert result(blocking=True, timeout=2) == 7
+    try:
+        assert result(blocking=True, timeout=10) == 7
+
+    except HueyException as error:
+        if "timed out" in str(error):
+            output = read_all_ready_output(huey_socket_consumer.stdout)
+            errorstream = read_all_ready_output(huey_socket_consumer.stderr)
+            raise TimeoutError(
+                f"Huey timed out: {str(error)}",
+                f"Consumer output: {output}",
+                f"Consumer errorstream: {errorstream}",
+            )
+        else:
+            raise error
 
 
 @pytest.mark.xfail(reason="cannot yet schedule with sockets for some reason")
 def test_huey_schedule(huey_queues_socket):
-    # we do not schedule tasks anywhere, but concerning.
-    result = sum.schedule((2, 5), delay=10)
+    # We do not schedule tasks anywhere, but concerning that it fails.
+    sum.schedule((2, 5), delay=10)
+
     try:
         assert len(huey.scheduled()) == 1
-    except assertionerror:
-        raise valueerror("have wrong amount of scheduled tasks", huey.scheduled())
+    except AssertionError:
+        raise ValueError("have wrong amount of scheduled tasks", huey.scheduled())

From 9ee72c1fcbef47dd44a5769c0b2b14df9202369a Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 20 Mar 2024 09:02:10 +0000
Subject: [PATCH 8/8] test(huey): make timeout more so that vm gets it in time

---
 tests/test_huey.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_huey.py b/tests/test_huey.py
index b14be9a..c741ae6 100644
--- a/tests/test_huey.py
+++ b/tests/test_huey.py
@@ -106,7 +106,7 @@ def test_huey_over_redis_socket(huey_socket_consumer):
 
     result = sum(2, 5)
     try:
-        assert result(blocking=True, timeout=10) == 7
+        assert result(blocking=True, timeout=100) == 7
 
     except HueyException as error:
         if "timed out" in str(error):