Fix Zocalo system tests, update zocalo environment (DiamondLightSourc…

…e#885)

* Add constant for zocalo environment constant with new names
* Fix Zocalo system tests (DiamondLightSource#893)

tests/conftest.py → conftest.py

@@ -44,6 +44,7 @@
 }
 
 BANNED_PATHS = [Path("/dls"), Path("/dls_sw")]
+environ["DODAL_TEST_MODE"] = "true"
 
 
 @pytest.fixture(autouse=True)
@@ -155,19 +156,6 @@ async def static_path_provider(
     return svpp
 
 
-@pytest.fixture
-async def RE():
-    RE = RunEngine()
-    # make sure the event loop is thoroughly up and running before we try to create
-    # any ophyd_async devices which might need it
-    timeout = time.monotonic() + 1
-    while not RE.loop.is_running():
-        await asyncio.sleep(0)
-        if time.monotonic() > timeout:
-            raise TimeoutError("This really shouldn't happen but just in case...")
-    yield RE
-
-
 @pytest.fixture
 def run_engine_documents(RE: RunEngine) -> Mapping[str, list[dict]]:
     docs: dict[str, list[dict]] = {}
@@ -185,3 +173,16 @@ def failed_status(failure: Exception) -> Status:
     status = Status()
     status.set_exception(failure)
     return status
+
+
+@pytest.fixture
+async def RE():
+    RE = RunEngine()
+    # make sure the event loop is thoroughly up and running before we try to create
+    # any ophyd_async devices which might need it
+    timeout = time.monotonic() + 1
+    while not RE.loop.is_running():
+        await asyncio.sleep(0)
+        if time.monotonic() > timeout:
+            raise TimeoutError("This really shouldn't happen but just in case...")
+    yield RE

src/dodal/devices/zocalo/zocalo_constants.py

@@ -0,0 +1,3 @@
+from dodal.utils import is_test_mode
+
+ZOCALO_ENV = "dev_bluesky" if is_test_mode() else "bluesky"

src/dodal/devices/zocalo/zocalo_interaction.py

@@ -7,6 +7,7 @@
 import zocalo.configuration
 from workflows.transport import lookup
 
+from dodal.devices.zocalo.zocalo_constants import ZOCALO_ENV
 from dodal.log import LOGGER
 
 
@@ -56,7 +57,7 @@ class ZocaloTrigger:
 
     see https://github.com/DiamondLightSource/dodal/wiki/How-to-Interact-with-Zocalo"""
 
-    def __init__(self, environment: str = "artemis"):
+    def __init__(self, environment: str = ZOCALO_ENV):
         self.zocalo_environment: str = environment
 
     def _send_to_zocalo(self, parameters: dict):

src/dodal/devices/zocalo/zocalo_results.py

@@ -21,6 +21,7 @@
 )
 from workflows.transport.common_transport import CommonTransport
 
+from dodal.devices.zocalo.zocalo_constants import ZOCALO_ENV
 from dodal.devices.zocalo.zocalo_interaction import _get_zocalo_connection
 from dodal.log import LOGGER
 
@@ -115,7 +116,7 @@ class ZocaloResults(StandardReadable, Triggerable):
     def __init__(
         self,
         name: str = "zocalo",
-        zocalo_environment: str = "dev_artemis",
+        zocalo_environment: str = ZOCALO_ENV,
         channel: str = "xrc.i03",
         sort_key: str = DEFAULT_SORT_KEY.value,
         timeout_s: float = DEFAULT_TIMEOUT,

src/dodal/utils.py

@@ -79,6 +79,10 @@ def get_beamline_name(default: str) -> str:
     return environ.get("BEAMLINE") or default
 
 
+def is_test_mode() -> bool:
+    return environ.get("DODAL_TEST_MODE") == "true"
+
+
 def get_hostname() -> str:
     return socket.gethostname().split(".")[0]
 

system_tests/test_zocalo_results.py

@@ -1,10 +1,10 @@
 import asyncio
-import os
 
 import bluesky.plan_stubs as bps
 import psutil
 import pytest
 from bluesky.preprocessors import stage_decorator
+from bluesky.protocols import Reading
 from bluesky.run_engine import RunEngine
 from bluesky.utils import FailedStatus
 
@@ -16,6 +16,7 @@
     ZocaloStartInfo,
     ZocaloTrigger,
 )
+from dodal.devices.zocalo.zocalo_constants import ZOCALO_ENV
 
 TEST_RESULT_LARGE: XrcResult = {
     "centre_of_mass": [1, 2, 3],
@@ -26,6 +27,8 @@
     "bounding_box": [[2, 2, 2], [8, 8, 7]],
 }
 
+DCID_WHICH_EXISTS_IN_DEV_ISPYB = 1000
+
 
 @pytest.fixture
 async def zocalo_device():
@@ -34,15 +37,35 @@ async def zocalo_device():
     return zd
 
 
+def convert_zocalo_device_reading_to_xrc_result(
+    zocalo_reading: dict[str, Reading],
+) -> XrcResult:
+    com = zocalo_reading["zocalo-centre_of_mass"]["value"][0]
+    max_voxel = zocalo_reading["zocalo-max_voxel"]["value"][0]
+    max_count = zocalo_reading["zocalo-max_count"]["value"][0]
+    n_voxels = zocalo_reading["zocalo-n_voxels"]["value"][0]
+    total_count = zocalo_reading["zocalo-total_count"]["value"][0]
+    bounding_box = zocalo_reading["zocalo-bounding_box"]["value"][0]
+
+    return XrcResult(
+        centre_of_mass=com.tolist(),
+        max_voxel=max_voxel.tolist(),
+        max_count=int(max_count),
+        n_voxels=int(n_voxels),
+        total_count=int(total_count),
+        bounding_box=bounding_box.tolist(),
+    )
+
+
 @pytest.mark.s03
 async def test_read_results_from_fake_zocalo(
     zocalo_device: ZocaloResults, RE: RunEngine
 ):
     zocalo_device._subscribe_to_results()
-    zc = ZocaloTrigger("dev_artemis")
-    zc.run_start(ZocaloStartInfo(0, None, 0, 100, 0))
-    zc.run_end(0)
-    zocalo_device.timeout_s = 5
+    zc = ZocaloTrigger(ZOCALO_ENV)
+    zc.run_start(ZocaloStartInfo(DCID_WHICH_EXISTS_IN_DEV_ISPYB, None, 0, 100, 0))
+    zc.run_end(DCID_WHICH_EXISTS_IN_DEV_ISPYB)
+    zocalo_device.timeout_s = 15
 
     def plan():
         yield from bps.open_run()
@@ -52,21 +75,21 @@ def plan():
     RE(plan())
 
     results = await zocalo_device.read()
-    assert results["zocalo-results"]["value"][0] == TEST_RESULT_LARGE
+    assert convert_zocalo_device_reading_to_xrc_result(results) == TEST_RESULT_LARGE
 
 
 @pytest.mark.s03
 async def test_stage_unstage_controls_read_results_from_fake_zocalo(
     zocalo_device: ZocaloResults, RE: RunEngine
 ):
     dodal.devices.zocalo.zocalo_results.CLEAR_QUEUE_WAIT_S = 0.05
-    zc = ZocaloTrigger("dev_artemis")
-    zocalo_device.timeout_s = 5
+    zc = ZocaloTrigger(ZOCALO_ENV)
+    zocalo_device.timeout_s = 15
 
     def plan():
         yield from bps.open_run()
-        zc.run_start(ZocaloStartInfo(0, None, 0, 100, 0))
-        zc.run_end(0)
+        zc.run_start(ZocaloStartInfo(DCID_WHICH_EXISTS_IN_DEV_ISPYB, None, 0, 100, 0))
+        zc.run_end(DCID_WHICH_EXISTS_IN_DEV_ISPYB)
         yield from bps.sleep(0.15)
         yield from bps.trigger_and_read([zocalo_device])
         yield from bps.close_run()
@@ -89,7 +112,7 @@ def plan_with_stage():
     await asyncio.sleep(1)
 
     results = await zocalo_device.read()
-    assert results["zocalo-results"]["value"][0] == TEST_RESULT_LARGE
+    assert convert_zocalo_device_reading_to_xrc_result(results) == TEST_RESULT_LARGE
     await zocalo_device.unstage()
 
     # Generating some more results leaves them at RMQ
@@ -104,16 +127,14 @@ def plan_with_stage():
 async def test_stale_connections_closed_after_unstage(
     zocalo_device: ZocaloResults, RE: RunEngine
 ):
-    this_process = psutil.Process(os.getpid())
-
-    connections_before = len(this_process.connections())
+    connections_before = len(psutil.net_connections())
 
     def stage_unstage():
         yield from bps.stage(zocalo_device)
         yield from bps.unstage(zocalo_device)
 
     RE(stage_unstage())
 
-    connections_after = len(this_process.connections())
+    connections_after = len(psutil.net_connections())
 
     assert connections_before == connections_after

tests/common/beamlines/test_beamline_utils.py

@@ -3,6 +3,7 @@
 
 import pytest
 from bluesky.run_engine import RunEngine as RE
+from conftest import mock_beamline_module_filepaths
 from ophyd import Device
 from ophyd.device import Device as OphydV1Device
 from ophyd.sim import FakeEpicsSignal
@@ -18,8 +19,6 @@
 from dodal.log import LOGGER
 from dodal.utils import make_all_devices
 
-from ...conftest import mock_beamline_module_filepaths
-
 
 @pytest.fixture(autouse=True)
 def flush_event_loop_on_finish(event_loop):

tests/devices/unit_tests/test_eiger.py

@@ -4,6 +4,7 @@
 from unittest.mock import ANY, MagicMock, Mock, call, create_autospec, patch
 
 import pytest
+from conftest import failed_status
 from ophyd.sim import NullStatus, make_fake_device
 from ophyd.status import Status
 from ophyd.utils import UnknownStatusFailure
@@ -15,8 +16,6 @@
 from dodal.devices.util.epics_util import run_functions_without_blocking
 from dodal.log import LOGGER
 
-from ...conftest import failed_status
-
 TEST_DETECTOR_SIZE_CONSTANTS = EIGER2_X_16M_SIZE
 
 TEST_EXPECTED_ENERGY = 100.0

tests/devices/unit_tests/test_undulator_dcm.py

@@ -3,6 +3,7 @@
 
 import numpy as np
 import pytest
+from conftest import MOCK_DAQ_CONFIG_PATH
 from ophyd_async.core import (
     AsyncStatus,
     DeviceCollector,
@@ -19,8 +20,6 @@
 )
 from dodal.log import LOGGER
 
-from ...conftest import MOCK_DAQ_CONFIG_PATH
-
 ID_GAP_LOOKUP_TABLE_PATH: str = (
     "./tests/devices/unit_tests/test_beamline_undulator_to_gap_lookup_table.txt"
 )

tests/devices/unit_tests/test_zocalo_interaction.py

@@ -11,7 +11,7 @@
 )
 from dodal.devices.zocalo.zocalo_interaction import ZocaloStartInfo
 
-SIM_ZOCALO_ENV = "dev_artemis"
+SIM_ZOCALO_ENV = "dev_bluesky"
 
 EXPECTED_DCID = 100
 EXPECTED_FILENAME = "test/file"

tests/devices/unit_tests/test_zocalo_results.py

@@ -11,6 +11,7 @@
 from bluesky.utils import FailedStatus
 from ophyd_async.core import AsyncStatus
 
+from dodal.devices.zocalo.zocalo_constants import ZOCALO_ENV
 from dodal.devices.zocalo.zocalo_results import (
     ZOCALO_READING_PLAN_NAME,
     NoResultsFromZocalo,
@@ -91,7 +92,7 @@ async def zocalo_results():
         patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"),
         patch("dodal.devices.zocalo.zocalo_results.CLEAR_QUEUE_WAIT_S", 0),
     ):
-        yield ZocaloResults(name="zocalo", zocalo_environment="dev_artemis")
+        yield ZocaloResults(name="zocalo", zocalo_environment=ZOCALO_ENV)
 
 
 @pytest.fixture
@@ -187,7 +188,7 @@ async def test_subscribe_only_on_called_stage(
     mock_connection: MagicMock, mock_wrap_subscribe: MagicMock, RE: RunEngine
 ):
     zocalo_results = ZocaloResults(
-        name="zocalo", zocalo_environment="dev_artemis", timeout_s=0
+        name="zocalo", zocalo_environment=ZOCALO_ENV, timeout_s=0
     )
     mock_wrap_subscribe.assert_not_called()
     await zocalo_results.stage()
@@ -212,7 +213,7 @@ async def test_zocalo_results_trigger_log_message(
 ):
     zocalo_results = ZocaloResults(
         name="zocalo",
-        zocalo_environment="dev_artemis",
+        zocalo_environment=ZOCALO_ENV,
         timeout_s=0,
         use_cpu_and_gpu=True,
     )
@@ -262,7 +263,7 @@ async def test_when_exception_caused_by_zocalo_message_then_exception_propagated
     mock_connection, RE: RunEngine
 ):
     zocalo_results = ZocaloResults(
-        name="zocalo", zocalo_environment="dev_artemis", timeout_s=0.1
+        name="zocalo", zocalo_environment=ZOCALO_ENV, timeout_s=0.1
     )
     await zocalo_results.connect()
     with pytest.raises(FailedStatus) as e:
@@ -431,7 +432,7 @@ async def test_gpu_results_ignored_and_cpu_results_used_if_toggle_disabled(
 ):
     zocalo_results = ZocaloResults(
         name="zocalo",
-        zocalo_environment="dev_artemis",
+        zocalo_environment=ZOCALO_ENV,
         timeout_s=0,
         use_cpu_and_gpu=False,
     )