From 37db4a4f4bdfbd6575cbd516aa19877b9cccff5c Mon Sep 17 00:00:00 2001 From: CoolFanyu Date: Tue, 24 Mar 2026 18:01:40 -0700 Subject: [PATCH 1/3] Modify gain acquisition Modified the trial function so the logic match with DE-Server. Correct the property name. Add parameter num_acq in take_gain_acquisition so user could save time for specific purpose. --- deapi/client.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/deapi/client.py b/deapi/client.py index 94447f8..b66aab0 100644 --- a/deapi/client.py +++ b/deapi/client.py @@ -2352,7 +2352,7 @@ def take_trial_gain_reference( if counting: self["Image Processing - Mode"] = "Counting" - self["Reference - Counting Gain Target (ADU/pix)"] = ( + self["Reference - Counting Gain Target (e-/pix)"] = ( 2000 if target_electrons_per_pixel is None else target_electrons_per_pixel @@ -2374,16 +2374,20 @@ def take_trial_gain_reference( while self.acquiring: time.sleep(2) + img, dtype, attr, _ = self.get_result(FrameType.SUMINTERMEDIATE, PixelFormat.UINT16) + if counting: exposure_time = self["Reference - Counting Gain Exposure Time (seconds)"] total_acquisitions = self["Reference - Counting Gain Acquisitions"] + num_el = np.max([attr.eppixpf * frame_rate, attr.eppixps]) + log.info(f"The number of electrons per pixel per second (eppixps): {num_el:.2f}") else: exposure_time = self["Reference - Integrating Gain Exposure Time (seconds)"] total_acquisitions = self["Reference - Integrating Gain Acquisitions"] + num_el = attr.imageMean * frame_rate + log.info(f"The number of ADUs per pixel per second : {num_el:.2f}") - img, dtype, attr, _ = self.get_result(FrameType.SUMTOTAL, PixelFormat.FLOAT32) self.SetProperty("Exposure Mode", prevExposureMode) - self.SetProperty("Exposure Time (seconds)", prevExposureTime) num_el = np.max([attr.eppixpf * frame_rate, attr.eppixps]) @@ -2396,10 +2400,11 @@ def take_trial_gain_reference( "The trial gain reference image has pixels that are close to saturation. " "Please reduce the beam intensity or exposure time." ) + # recalculating to check... - total_acquisitions = int( - np.ceil(target_electrons_per_pixel / (exposure_time * num_el)) - ) + # total_acquisitions = int( + # np.ceil(target_electrons_per_pixel / (exposure_time * num_el)) + # ) if total_acquisitions == 1: total_acquisitions = 2 @@ -2413,6 +2418,7 @@ def take_gain_reference( target_electrons_per_pixel: float = None, timeout: int = 600, counting: bool = False, + num_acq: int = 0 ): """Take a gain reference. @@ -2439,6 +2445,8 @@ def take_gain_reference( If True, the gain reference will be taken in counting mode, by default False. This is useful for cameras that support counting mode and can be used to take gain references with a lower noise level. + num_acq: int, optional + Force the number of acquisiton to this number, otherwise it will be automatically calculated. """ if target_electrons_per_pixel is None and not counting: target_electrons_per_pixel = 16000 @@ -2449,6 +2457,9 @@ def take_gain_reference( frame_rate, target_electrons_per_pixel, counting ) + if (num_acq != 0): + num_acquisitions = num_acq + log.info( f"Gain reference: {exposure_time:.2f} seconds, " f"total acquisitions: {num_acquisitions}, " From bdf24a864dbec721aa8038509f2ab5da0e102075 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Fri, 3 Apr 2026 09:29:13 -0500 Subject: [PATCH 2/3] Enhance socket communication and add wait_for_idle utility for improved test reliability --- deapi/client.py | 34 ++++++++++++------ deapi/simulated_server/initialize_server.py | 5 +++ deapi/tests/conftest.py | 31 +++++++++++++++++ deapi/tests/test_client.py | 38 +++++++-------------- 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/deapi/client.py b/deapi/client.py index b66aab0..78430d5 100644 --- a/deapi/client.py +++ b/deapi/client.py @@ -2374,13 +2374,17 @@ def take_trial_gain_reference( while self.acquiring: time.sleep(2) - img, dtype, attr, _ = self.get_result(FrameType.SUMINTERMEDIATE, PixelFormat.UINT16) + img, dtype, attr, _ = self.get_result( + FrameType.SUMINTERMEDIATE, PixelFormat.UINT16 + ) if counting: exposure_time = self["Reference - Counting Gain Exposure Time (seconds)"] total_acquisitions = self["Reference - Counting Gain Acquisitions"] num_el = np.max([attr.eppixpf * frame_rate, attr.eppixps]) - log.info(f"The number of electrons per pixel per second (eppixps): {num_el:.2f}") + log.info( + f"The number of electrons per pixel per second (eppixps): {num_el:.2f}" + ) else: exposure_time = self["Reference - Integrating Gain Exposure Time (seconds)"] total_acquisitions = self["Reference - Integrating Gain Acquisitions"] @@ -2403,7 +2407,7 @@ def take_trial_gain_reference( # recalculating to check... # total_acquisitions = int( - # np.ceil(target_electrons_per_pixel / (exposure_time * num_el)) + # np.ceil(target_electrons_per_pixel / (exposure_time * num_el)) # ) if total_acquisitions == 1: total_acquisitions = 2 @@ -2418,7 +2422,7 @@ def take_gain_reference( target_electrons_per_pixel: float = None, timeout: int = 600, counting: bool = False, - num_acq: int = 0 + num_acq: int = 0, ): """Take a gain reference. @@ -2457,7 +2461,7 @@ def take_gain_reference( frame_rate, target_electrons_per_pixel, counting ) - if (num_acq != 0): + if num_acq != 0: num_acquisitions = num_acq log.info( @@ -2722,18 +2726,28 @@ def _sendCommand(self, command: pb.DEPacket = None): try: packet = struct.pack("I", command.ByteSize()) + command.SerializeToString() - res = self.socket.send(packet) - # packet.PrintDebugString() - # log.debug("sent result = %d\n", res) + # sendall() loops internally until every byte is delivered (or raises). + # send() can return a short count on a non-blocking/timeout socket — + # that leaves _recv_exact on the server waiting for the rest of the + # message while this side waits for the reply: a deadlock. + self.socket.sendall(packet) except socket.error as e: - raise e("Error sending %s\n", command) + log.error("Error sending command: %s", e) + return False if logLevel == logging.DEBUG: lapsed = (self.GetTime() - step_time) * 1000 log.debug(" Send Time: %.1f ms", lapsed) step_time = self.GetTime() - return self.__ReceiveResponseForCommand(command) + try: + return self.__ReceiveResponseForCommand(command) + except ConnectionResetError as e: + # Server closed the connection mid-reply (e.g. it crashed or dropped + # the client). Return False so callers' existing `if response != False` + # guards work correctly, rather than propagating an unexpected exception. + log.error("Connection reset while waiting for response: %s", e) + return False def __ReceiveResponseForCommand(self, command): step_time = self.GetTime() diff --git a/deapi/simulated_server/initialize_server.py b/deapi/simulated_server/initialize_server.py index 2483c17..ce53321 100644 --- a/deapi/simulated_server/initialize_server.py +++ b/deapi/simulated_server/initialize_server.py @@ -54,6 +54,11 @@ def main(port=13240): sys.stderr.flush() while True: conn, addr = server_socket.accept() # What waits for a connection + # Guard against a stalled client leaving _recv_exact blocked forever. + # 120 s is generous enough to survive debugger pauses and slow CI + # runners, while still breaking the partial-send deadlock if sendall() + # somehow delivers a short write. + conn.settimeout(120) server = FakeServer(socket=conn) connected = True while connected: diff --git a/deapi/tests/conftest.py b/deapi/tests/conftest.py index c057170..660a533 100644 --- a/deapi/tests/conftest.py +++ b/deapi/tests/conftest.py @@ -22,6 +22,37 @@ def close_port(port): process.terminate() +def wait_for_idle(client, timeout: float = 30, interval: float = 0.1): + """Poll ``client.acquiring`` until it is False or *timeout* seconds elapse. + + Replaces bare ``while client.acquiring: time.sleep(N)`` loops in tests so + that a stalled socket or an unexpectedly long acquisition does not cause the + test suite to hang indefinitely. + + The default *timeout* of 30 s is sized for fake-server tests where all + acquisitions complete in well under a second (FPS=1000, small scan grids). + Pass a larger value for tests that run against real hardware with long + exposures, e.g. ``wait_for_idle(client, timeout=300)``. + + Parameters + ---------- + client: + A connected :class:`deapi.Client` instance. + timeout: + Maximum number of seconds to wait before failing the test. + interval: + Polling interval in seconds. + """ + deadline = time.monotonic() + timeout + while client.acquiring: + if time.monotonic() > deadline: + pytest.fail( + f"Camera still acquiring after {timeout:.0f} s — " + "possible socket deadlock or FakeServer state error" + ) + time.sleep(interval) + + # Modifying pytest run options def pytest_addoption(parser): parser.addoption( diff --git a/deapi/tests/test_client.py b/deapi/tests/test_client.py index 9b194e9..3de70cd 100644 --- a/deapi/tests/test_client.py +++ b/deapi/tests/test_client.py @@ -10,6 +10,7 @@ MovieBufferStatus, ContrastStretchType, ) +from deapi.tests.conftest import wait_for_idle class TestClient: @@ -57,8 +58,7 @@ def test_start_acquisition(self, client): client.scan(size_x=10, size_y=10, enable="On") client.start_acquisition(1) assert client.acquiring - while client.acquiring: - time.sleep(1) + wait_for_idle(client) assert not client.acquiring def test_start_acquisition_scan_disabled(self, client): @@ -66,8 +66,7 @@ def test_start_acquisition_scan_disabled(self, client): client.scan(enable="Off") client.start_acquisition(10) assert client.acquiring - while client.acquiring: - time.sleep(1) + wait_for_idle(client) assert not client.acquiring def test_get_result(self, client): @@ -80,8 +79,7 @@ def test_get_result(self, client): assert client["Hardware ROI Offset X"] == 0 assert client["Hardware ROI Offset Y"] == 0 client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) result = client.get_result() assert isinstance(result, tuple) assert len(result) == 4 @@ -93,8 +91,7 @@ def test_get_histogram(self, client): client["Frames Per Second"] = 1000 client.scan(size_x=10, size_y=10, enable="On") client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) result = client.get_result("singleframe_integrated") assert isinstance(result[3], Histogram) result[3].plot() @@ -107,8 +104,7 @@ def test_get_result_no_scan(self, client): assert isinstance(result, tuple) assert len(result) == 4 assert result[0].shape == (1024, 1024) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) def test_binning_linked_parameters(self, client): @@ -123,8 +119,7 @@ def test_binning(self, client, binx): client["Hardware Binning X"] = binx assert client["Hardware Binning X"] == binx client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) result = client.get_result("singleframe_integrated") assert result[0].shape[1] == 1024 // binx @@ -173,8 +168,7 @@ def test_virtual_mask_calculation(self, client): assert client["Scan - Virtual Detector 3 Calculation"] == "Difference" np.testing.assert_allclose(client.virtual_masks[2][::2], 2) client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) result = client.get_result("virtual_image3") assert result is not None assert result[0].shape == (10, 8) @@ -309,8 +303,7 @@ def test_set_xy_array(self, client): assert is_set assert client["Scan - Points"] == np.sum(mask) * 2 client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) result = client.get_result("virtual_image1") assert result[0].shape == (12, 12) client["Scan - Type"] = "Raster" # clean up @@ -352,28 +345,21 @@ def test_flip_dark_reference(self, client): client.take_dark_reference(frame_rate=10) client["Image Processing - Flatfield Correction"] = "Dark" client.start_acquisition(1) - # assert that the dark reference corrects the image to zero... - while client.acquiring: - time.sleep(1) + wait_for_idle(client) image = client.get_result()[0] np.testing.assert_array_equal(image, 0) - # Now flip the dark reference client["Image Processing - Flip Horizontally"] = "On" client["Exposure Time (seconds)"] = 1 client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) image = client.get_result()[0] np.testing.assert_array_equal(image, 0) client["Image Processing - Flip Horizontally"] = "Off" - # test bin by a factor of 2 - client["Binning X"] = 2 client["Binning Y"] = 2 client.start_acquisition(1) - while client.acquiring: - time.sleep(1) + wait_for_idle(client) image = client.get_result()[0] np.testing.assert_array_equal(image, 0) From 84eb7f7e55a69c1bfaf21ac61cbc19646d1651b8 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Fri, 3 Apr 2026 14:35:20 -0500 Subject: [PATCH 3/3] Remove macOS from OS matrix in build configuration --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6093a8b..6cd65d6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest] python-version: ["3.12", "3.13", "3.14"] include: - os: ubuntu-latest