fix(huey): adapt to new VM test environment
continuous-integration/drone/push Build is failing
Details
continuous-integration/drone/push Build is failing
Details
parent
baf7843349
commit
6f38b2309f
|
@ -27,6 +27,7 @@
|
||||||
python-lsp-server
|
python-lsp-server
|
||||||
pyflakes
|
pyflakes
|
||||||
typer # for strawberry
|
typer # for strawberry
|
||||||
|
types-redis # for mypy
|
||||||
] ++ strawberry-graphql.optional-dependencies.cli));
|
] ++ strawberry-graphql.optional-dependencies.cli));
|
||||||
|
|
||||||
vmtest-src-dir = "/root/source";
|
vmtest-src-dir = "/root/source";
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
"""
|
"""
|
||||||
Redis pool module for selfprivacy_api
|
Redis pool module for selfprivacy_api
|
||||||
"""
|
"""
|
||||||
from os import environ
|
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
|
from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
|
||||||
|
|
||||||
|
REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
|
||||||
|
|
||||||
|
|
||||||
class RedisPool(metaclass=SingletonMetaclass):
|
class RedisPool(metaclass=SingletonMetaclass):
|
||||||
"""
|
"""
|
||||||
|
@ -24,19 +26,7 @@ class RedisPool(metaclass=SingletonMetaclass):
|
||||||
redis://[[username]:[password]]@localhost:6379/0
|
redis://[[username]:[password]]@localhost:6379/0
|
||||||
unix://[username@]/path/to/socket.sock?db=0[&password=password]
|
unix://[username@]/path/to/socket.sock?db=0[&password=password]
|
||||||
"""
|
"""
|
||||||
|
return f"unix://{REDIS_SOCKET}?db={dbnumber}"
|
||||||
if "USE_REDIS_PORT" in environ:
|
|
||||||
port = int(environ["USE_REDIS_PORT"])
|
|
||||||
return f"redis://@127.0.0.1:{port}/{dbnumber}"
|
|
||||||
else:
|
|
||||||
return f"unix://{RedisPool.redis_socket()}?db={dbnumber}"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def redis_socket() -> str:
|
|
||||||
if "REDIS_SOCKET" in environ:
|
|
||||||
return environ["REDIS_SOCKET"]
|
|
||||||
else:
|
|
||||||
return "/run/redis-sp-api/redis.sock"
|
|
||||||
|
|
||||||
def get_connection(self):
|
def get_connection(self):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -1,14 +1,15 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from subprocess import Popen
|
|
||||||
from os import environ, path
|
|
||||||
import redis
|
import redis
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from subprocess import Popen, check_output, TimeoutExpired
|
||||||
|
from os import environ, path, set_blocking
|
||||||
|
from io import BufferedReader
|
||||||
|
from huey.exceptions import HueyException
|
||||||
|
|
||||||
from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
|
from selfprivacy_api.utils.huey import huey, immediate, HUEY_DATABASE_NUMBER
|
||||||
from selfprivacy_api.utils.redis_pool import RedisPool
|
from selfprivacy_api.utils.redis_pool import RedisPool, REDIS_SOCKET
|
||||||
from selfprivacy_api.utils.waitloop import wait_until_true
|
|
||||||
|
|
||||||
from selfprivacy_api.backup.util import output_yielder
|
|
||||||
|
|
||||||
|
|
||||||
@huey.task()
|
@huey.task()
|
||||||
|
@ -28,66 +29,28 @@ def flush_huey_redis_forcefully():
|
||||||
connection.flushdb()
|
connection.flushdb()
|
||||||
|
|
||||||
|
|
||||||
def start_redis_socket(socket_path):
|
# TODO: may be useful in other places too, move to utils/ tests common if using it somewhere
|
||||||
# Socket file will be created by redis
|
def read_all_ready_output(stream: BufferedReader) -> str:
|
||||||
command = [
|
set_blocking(stream.fileno(), False)
|
||||||
"redis-server",
|
output: List[bytes] = []
|
||||||
"--unixsocket",
|
while True:
|
||||||
socket_path,
|
line = stream.readline()
|
||||||
"--unixsocketperm",
|
raise ValueError(line)
|
||||||
"700",
|
if line == b"":
|
||||||
"--port",
|
break
|
||||||
"0",
|
else:
|
||||||
]
|
output.append(line)
|
||||||
redis_handle = Popen(command)
|
|
||||||
|
|
||||||
wait_until_true(lambda: path.exists(socket_path), timeout_sec=2)
|
set_blocking(stream.fileno(), True)
|
||||||
flush_huey_redis_forcefully()
|
|
||||||
|
|
||||||
return redis_handle
|
result = b"".join(output)
|
||||||
|
return result.decode("utf-8")
|
||||||
|
|
||||||
@pytest.fixture()
|
|
||||||
def redis_socket(tmpdir):
|
|
||||||
# Does NOT overwrite already imported redis pools
|
|
||||||
# -> Not very useful for more involved tests
|
|
||||||
# DOES override imported huey partially, but tries to restore it back
|
|
||||||
|
|
||||||
socket_path = path.join(tmpdir, "redis.sock")
|
|
||||||
environ["REDIS_SOCKET"] = socket_path
|
|
||||||
|
|
||||||
old_port = None
|
|
||||||
if "USE_REDIS_PORT" in environ:
|
|
||||||
old_port = environ["USE_REDIS_PORT"]
|
|
||||||
del environ["USE_REDIS_PORT"]
|
|
||||||
|
|
||||||
assert "USE_REDIS_PORT" not in environ
|
|
||||||
|
|
||||||
old_huey_url = huey.storage_kwargs.get("url")
|
|
||||||
# Overriding url in the already imported singleton
|
|
||||||
huey.storage_kwargs["url"] = RedisPool.connection_url(HUEY_DATABASE_NUMBER)
|
|
||||||
reset_huey_storage()
|
|
||||||
|
|
||||||
redis_handle = start_redis_socket(socket_path)
|
|
||||||
|
|
||||||
yield socket_path
|
|
||||||
|
|
||||||
# Socket file will be destroyed by redis
|
|
||||||
redis_handle.terminate()
|
|
||||||
|
|
||||||
if old_port:
|
|
||||||
environ["USE_REDIS_PORT"] = old_port
|
|
||||||
del environ["REDIS_SOCKET"]
|
|
||||||
|
|
||||||
if old_huey_url:
|
|
||||||
huey.storage_kwargs["url"] = old_huey_url
|
|
||||||
else:
|
|
||||||
del huey.storage_kwargs["url"]
|
|
||||||
reset_huey_storage()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def not_immediate():
|
def not_immediate():
|
||||||
|
assert environ["TEST_MODE"] == "true"
|
||||||
|
|
||||||
old_immediate = huey.immediate
|
old_immediate = huey.immediate
|
||||||
environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
|
environ["HUEY_QUEUES_FOR_TESTS"] = "Yes"
|
||||||
huey.immediate = False
|
huey.immediate = False
|
||||||
|
@ -101,48 +64,32 @@ def not_immediate():
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def huey_queues(not_immediate):
|
def huey_socket_consumer(not_immediate):
|
||||||
"""
|
|
||||||
Full, not-immediate, queued huey, with consumer starting and stopping.
|
|
||||||
IMPORTANT: Assumes tests are run from the project directory.
|
|
||||||
The above is needed by consumer to find our huey setup.
|
|
||||||
"""
|
|
||||||
flush_huey_redis_forcefully()
|
|
||||||
command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
|
|
||||||
consumer_handle = Popen(command)
|
|
||||||
|
|
||||||
yield huey
|
|
||||||
|
|
||||||
consumer_handle.kill()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
|
||||||
def huey_queues_socket(not_immediate, redis_socket):
|
|
||||||
"""
|
"""
|
||||||
Same as above, but with socketed redis
|
Same as above, but with socketed redis
|
||||||
"""
|
"""
|
||||||
|
|
||||||
flush_huey_redis_forcefully()
|
flush_huey_redis_forcefully()
|
||||||
command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
|
command = ["huey_consumer.py", "selfprivacy_api.task_registry.huey"]
|
||||||
consumer_handle = Popen(command)
|
|
||||||
|
|
||||||
assert path.exists(redis_socket)
|
# First assert that consumer does not fail by itself
|
||||||
|
# Idk yet how to do it more elegantly
|
||||||
|
try:
|
||||||
|
check_output(command, timeout=2)
|
||||||
|
except TimeoutExpired:
|
||||||
|
pass
|
||||||
|
|
||||||
yield redis_socket
|
# Then open it for real
|
||||||
|
consumer_handle = Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
|
||||||
|
assert path.exists(REDIS_SOCKET)
|
||||||
|
|
||||||
|
yield consumer_handle
|
||||||
|
|
||||||
consumer_handle.kill()
|
consumer_handle.kill()
|
||||||
|
|
||||||
|
|
||||||
def test_huey_over_redis(huey_queues):
|
def test_huey_over_redis_socket(huey_socket_consumer):
|
||||||
assert huey.immediate is False
|
|
||||||
assert immediate() is False
|
|
||||||
|
|
||||||
result = sum(2, 5)
|
|
||||||
assert result(blocking=True, timeout=2) == 7
|
|
||||||
|
|
||||||
|
|
||||||
# we cannot have these two fixtures prepared at the same time to iterate through them
|
|
||||||
def test_huey_over_redis_socket(huey_queues_socket):
|
|
||||||
assert huey.immediate is False
|
assert huey.immediate is False
|
||||||
assert immediate() is False
|
assert immediate() is False
|
||||||
|
|
||||||
|
@ -158,14 +105,28 @@ def test_huey_over_redis_socket(huey_queues_socket):
|
||||||
)
|
)
|
||||||
|
|
||||||
result = sum(2, 5)
|
result = sum(2, 5)
|
||||||
assert result(blocking=True, timeout=2) == 7
|
try:
|
||||||
|
assert result(blocking=True, timeout=10) == 7
|
||||||
|
|
||||||
|
except HueyException as error:
|
||||||
|
if "timed out" in str(error):
|
||||||
|
output = read_all_ready_output(huey_socket_consumer.stdout)
|
||||||
|
errorstream = read_all_ready_output(huey_socket_consumer.stderr)
|
||||||
|
raise TimeoutError(
|
||||||
|
f"Huey timed out: {str(error)}",
|
||||||
|
f"Consumer output: {output}",
|
||||||
|
f"Consumer errorstream: {errorstream}",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise error
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail(reason="cannot yet schedule with sockets for some reason")
|
@pytest.mark.xfail(reason="cannot yet schedule with sockets for some reason")
|
||||||
def test_huey_schedule(huey_queues_socket):
|
def test_huey_schedule(huey_queues_socket):
|
||||||
# we do not schedule tasks anywhere, but concerning.
|
# We do not schedule tasks anywhere, but concerning that it fails.
|
||||||
result = sum.schedule((2, 5), delay=10)
|
sum.schedule((2, 5), delay=10)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
assert len(huey.scheduled()) == 1
|
assert len(huey.scheduled()) == 1
|
||||||
except assertionerror:
|
except AssertionError:
|
||||||
raise valueerror("have wrong amount of scheduled tasks", huey.scheduled())
|
raise ValueError("have wrong amount of scheduled tasks", huey.scheduled())
|
||||||
|
|
Loading…
Reference in New Issue