From e9231c878de3dfca9a16e4f7889658f1c6038151 Mon Sep 17 00:00:00 2001 From: Tim Huff Date: Tue, 7 Apr 2026 18:53:26 +0000 Subject: [PATCH 1/6] first pass --- test/integration/test_groundlight.py | 23 ++++++++++++++++ test/retry_decorator.py | 30 ++++++++++++++++++++ test/unit/test_customizable_request.py | 7 ++--- test/unit/test_experimental.py | 4 +++ test/unit/test_images.py | 9 +++--- test/unit/test_internalapi.py | 10 ++++--- test/unit/test_labels.py | 35 +++++++++++++----------- test/unit/test_metrics_and_evaluation.py | 7 ++--- 8 files changed, 93 insertions(+), 32 deletions(-) create mode 100644 test/retry_decorator.py diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 6d69bb63..c7671e3a 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -26,6 +26,7 @@ PaginatedDetectorList, PaginatedImageQueryList, ) +from test.retry_decorator import retry_on_failure from urllib3.exceptions import ConnectTimeoutError, MaxRetryError, ReadTimeoutError from urllib3.util.retry import Retry @@ -273,6 +274,7 @@ def test_get_detector_by_name(gl: Groundlight, detector: Detector): gl.get_detector_by_name(name="not a real name") +@retry_on_failure() def test_ask_confident(gl: Groundlight, detector: Detector): _image_query = gl.ask_confident(detector=detector.id, image="test/assets/dog.jpeg", wait=10) assert str(_image_query) @@ -280,6 +282,7 @@ def test_ask_confident(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_ask_ml(gl: Groundlight, detector: Detector): _image_query = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=10) assert str(_image_query) @@ -287,6 +290,7 @@ def test_ask_ml(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_submit_image_query(gl: Groundlight, detector: Detector): def validate_image_query(_image_query: ImageQuery): assert str(_image_query) @@ -314,6 +318,7 @@ def validate_image_query(_image_query: ImageQuery): assert _image_query.result.confidence >= IQ_IMPROVEMENT_THRESHOLD +@retry_on_failure() def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): _image_query = gl.submit_image_query( detector=detector.id, image="test/assets/dog.jpeg", wait=10, human_review="NEVER" @@ -323,6 +328,7 @@ def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_submit_image_query_returns_yes(gl: Groundlight): # We use the "never-review" pipeline to guarantee a confident "yes" answer. detector = gl.get_or_create_detector(name="Always a dog", query="Is there a dog?", pipeline_config="never-review") @@ -330,6 +336,7 @@ def test_submit_image_query_returns_yes(gl: Groundlight): assert image_query.result.label == Label.YES +@retry_on_failure() def test_submit_image_query_returns_text(gl: Groundlight): # We use the "never-review" pipeline to guarantee a confident "yes" answer. detector = gl.get_or_create_detector( @@ -339,6 +346,7 @@ def test_submit_image_query_returns_text(gl: Groundlight): assert isinstance(image_query.text, str) +@retry_on_failure() def test_submit_image_query_filename(gl: Groundlight, detector: Detector): _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER") assert str(_image_query) @@ -346,6 +354,7 @@ def test_submit_image_query_filename(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_submit_image_query_png(gl: Groundlight, detector: Detector): _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/cat.png", human_review="NEVER") assert str(_image_query) @@ -353,6 +362,7 @@ def test_submit_image_query_png(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_submit_image_query_with_confidence_threshold(gl: Groundlight, detector: Detector): confidence_threshold = 0.5234 # Arbitrary specific value _image_query = gl.submit_image_query( @@ -366,6 +376,7 @@ def test_submit_image_query_with_confidence_threshold(gl: Groundlight, detector: @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing an image query ID.") +@retry_on_failure() def test_submit_image_query_with_id(gl: Groundlight, detector: Detector): # submit_image_query id = f"iq_{KsuidMs()}" @@ -380,6 +391,7 @@ def test_submit_image_query_with_id(gl: Groundlight, detector: Detector): assert _image_query.metadata.get("is_from_edge") +@retry_on_failure() def test_submit_image_query_with_human_review_param(gl: Groundlight, detector: Detector): # For now, this just tests that the image query is submitted successfully. # There should probably be a better way to check whether the image query was escalated for human review. @@ -451,6 +463,7 @@ def test_create_detector_with_invalid_metadata(gl: Groundlight, metadata_list: A @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing image query metadata.") @pytest.mark.parametrize("metadata", [None, {}, {"a": 1}, '{"a": 1}']) +@retry_on_failure() def test_submit_image_query_with_metadata( gl: Groundlight, detector: Detector, image: str, metadata: Union[Dict, str, None] ): @@ -505,6 +518,7 @@ def test_submit_image_query_with_metadata_returns_user_error(gl: Groundlight, de assert is_user_error(exc_info.value.status) +@retry_on_failure() def test_submit_image_query_jpeg_bytes(gl: Groundlight, detector: Detector): jpeg = open("test/assets/dog.jpeg", "rb").read() _image_query = gl.submit_image_query(detector=detector.id, image=jpeg, human_review="NEVER") @@ -543,6 +557,7 @@ def test_submit_image_query_bad_jpeg_file(gl: Groundlight, detector: Detector): @pytest.mark.skipif(MISSING_PIL, reason="Needs pillow") # type: ignore +@retry_on_failure() def test_submit_image_query_pil(gl: Groundlight, detector: Detector): # generates a pil image and submits it from PIL import Image @@ -565,6 +580,7 @@ def test_submit_image_query_wait_and_want_async_causes_exception(gl: Groundlight ) +@retry_on_failure() def test_submit_image_query_with_want_async_workflow(gl: Groundlight, detector: Detector): """ Tests the workflow for submitting an image query with the want_async parameter set to True. @@ -589,6 +605,7 @@ def test_submit_image_query_with_want_async_workflow(gl: Groundlight, detector: assert _image_query.result.label in VALID_DISPLAY_LABELS +@retry_on_failure() def test_ask_async_workflow(gl: Groundlight, detector: Detector): """ Tests the workflow for submitting an image query with ask_async. @@ -709,6 +726,7 @@ def test_enum_string_equality(): @pytest.mark.skipif(MISSING_NUMPY or MISSING_PIL, reason="Needs numpy and pillow") # type: ignore +@retry_on_failure() def test_submit_numpy_image(gl: Groundlight, detector: Detector): np_img = np.random.uniform(0, 255, (600, 800, 3)) # type: ignore _image_query = gl.submit_image_query(detector=detector.id, image=np_img, human_review="NEVER") @@ -820,6 +838,7 @@ def test_update_detector_confidence_threshold_failure(gl: Groundlight, detector: @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.") +@retry_on_failure() def test_submit_image_query_with_inspection_id_metadata_and_want_async(gl: Groundlight, detector: Detector, image: str): inspection_id = gl.start_inspection() metadata = {"key": "value"} @@ -852,6 +871,7 @@ def test_submit_image_query_with_empty_inspection_id(gl: Groundlight, detector: ) +@retry_on_failure() def test_binary_detector(gl: Groundlight, detector_name: Callable): """ verify that we can create and submit to a binary detector @@ -863,6 +883,7 @@ def test_binary_detector(gl: Groundlight, detector_name: Callable): assert binary_iq.result.label is not None +@retry_on_failure() def test_counting_detector(gl: Groundlight, detector_name: Callable): """ verify that we can create and submit to a counting detector @@ -874,6 +895,7 @@ def test_counting_detector(gl: Groundlight, detector_name: Callable): assert count_iq.result.count is not None +@retry_on_failure() def test_counting_detector_async(gl: Groundlight, detector_name: Callable): """ verify that we can create and submit to a counting detector @@ -893,6 +915,7 @@ def test_counting_detector_async(gl: Groundlight, detector_name: Callable): assert _image_query.result is not None +@retry_on_failure() def test_multiclass_detector(gl: Groundlight, detector_name: Callable): """ verify that we can create and submit to a multi-class detector diff --git a/test/retry_decorator.py b/test/retry_decorator.py new file mode 100644 index 00000000..a86b353a --- /dev/null +++ b/test/retry_decorator.py @@ -0,0 +1,30 @@ +"""Test-only helpers for retrying tests affected by transient cloud timing.""" + +import functools +from typing import Any, Callable, Tuple, Type + + +def retry_on_failure( + *, + max_attempts: int = 3, + exception_types: Tuple[Type[BaseException], ...] = (Exception,), +) -> Callable[[Callable[..., Any]], Callable[..., None]]: + """Run the wrapped test up to `max_attempts` times when it raises a listed exception.""" + + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + + def decorator(fn: Callable[..., Any]) -> Callable[..., None]: + @functools.wraps(fn) + def wrapper(*args: object, **kwargs: object) -> None: + for attempt in range(max_attempts): + try: + fn(*args, **kwargs) + return + except exception_types: + if attempt == max_attempts - 1: + raise + + return wrapper + + return decorator diff --git a/test/unit/test_customizable_request.py b/test/unit/test_customizable_request.py index 9f62d73c..16de4781 100644 --- a/test/unit/test_customizable_request.py +++ b/test/unit/test_customizable_request.py @@ -1,12 +1,11 @@ -from datetime import datetime +from typing import Callable from groundlight import ExperimentalApi gl = ExperimentalApi() -def test_invalid_endpoint_config(): +def test_invalid_endpoint_config(detector_name: Callable): print(gl.make_generic_api_request(endpoint="/v1/me", method="GET")) print(gl.make_generic_api_request(endpoint="/v1/detectors", method="GET")) - name = f"Test {datetime.utcnow()}" - print(gl.make_generic_api_request(endpoint="/v1/detector-groups", method="POST", body={"name": name})) + print(gl.make_generic_api_request(endpoint="/v1/detector-groups", method="POST", body={"name": detector_name()})) diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 9c2237e5..81223aa1 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -4,6 +4,7 @@ import pytest from groundlight import ExperimentalApi from model import Detector, ImageQuery +from test.retry_decorator import retry_on_failure def test_detector_groups(gl_experimental: ExperimentalApi, detector_name: Callable): @@ -90,6 +91,7 @@ def test_submit_multiple_rois(gl_experimental: ExperimentalApi, image_query_one: gl_experimental.add_label(image_query_one, 3, [roi] * 3) +@retry_on_failure() def test_text_recognition_detector(gl_experimental: ExperimentalApi, detector_name: Callable): """ verify that we can create and submit to a text recognition detector @@ -103,6 +105,7 @@ def test_text_recognition_detector(gl_experimental: ExperimentalApi, detector_na assert mc_iq.result.text is not None +@retry_on_failure() def test_bounding_box_detector(gl_experimental: ExperimentalApi, detector_name: Callable): """ Verify that we can create and submit to a bounding box detector @@ -117,6 +120,7 @@ def test_bounding_box_detector(gl_experimental: ExperimentalApi, detector_name: assert bbox_iq.rois is not None +@retry_on_failure() def test_bounding_box_detector_async(gl_experimental: ExperimentalApi, detector_name: Callable): """ Verify that we can create and submit to a bounding box detector with ask_async diff --git a/test/unit/test_images.py b/test/unit/test_images.py index 12220ade..249cb7be 100644 --- a/test/unit/test_images.py +++ b/test/unit/test_images.py @@ -1,12 +1,13 @@ -from datetime import datetime +from typing import Callable import PIL from groundlight import ExperimentalApi +from test.retry_decorator import retry_on_failure -def test_get_image(gl_experimental: ExperimentalApi): - name = f"Test {datetime.utcnow()}" - det = gl_experimental.get_or_create_detector(name, "test_query") +@retry_on_failure() +def test_get_image(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.get_or_create_detector(detector_name(), "test_query") iq = gl_experimental.submit_image_query(det, image="test/assets/dog.jpeg", wait=10) gl_experimental.get_image(iq.id) assert isinstance(PIL.Image.open(gl_experimental.get_image(iq.id)), PIL.Image.Image) diff --git a/test/unit/test_internalapi.py b/test/unit/test_internalapi.py index 5f3422de..870efea0 100644 --- a/test/unit/test_internalapi.py +++ b/test/unit/test_internalapi.py @@ -1,18 +1,20 @@ +from typing import Callable + from groundlight import ExperimentalApi from groundlight.internalapi import iq_is_answered, iq_is_confident from model import ImageQuery -def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): - det = gl_experimental.get_or_create_detector("Test", "test_query") +def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq: ImageQuery, detector_name: Callable): + det = gl_experimental.get_or_create_detector(detector_name(), "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_confident(iq, 0.9) assert not iq_is_confident(initial_iq, 0.9) -def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): - det = gl_experimental.get_or_create_detector("Test", "test_query") +def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq: ImageQuery, detector_name: Callable): + det = gl_experimental.get_or_create_detector(detector_name(), "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_answered(iq) diff --git a/test/unit/test_labels.py b/test/unit/test_labels.py index a504d1e4..90df9b9e 100644 --- a/test/unit/test_labels.py +++ b/test/unit/test_labels.py @@ -1,12 +1,13 @@ -from datetime import datetime +from typing import Callable import pytest from groundlight import ApiException, ExperimentalApi +from test.retry_decorator import retry_on_failure -def test_binary_labels(gl_experimental: ExperimentalApi): - name = f"Test binary labels{datetime.utcnow()}" - det = gl_experimental.create_detector(name, "test_query") +@retry_on_failure() +def test_binary_labels(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_detector(detector_name("Test binary labels"), "test_query") iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "YES") iq1 = gl_experimental.get_image_query(iq1.id) @@ -21,9 +22,9 @@ def test_binary_labels(gl_experimental: ExperimentalApi): gl_experimental.add_label(iq1, "MAYBE") -def test_counting_labels(gl_experimental: ExperimentalApi): - name = f"Test binary labels{datetime.utcnow()}" - det = gl_experimental.create_counting_detector(name, "test_query", "test_object_class") +@retry_on_failure() +def test_counting_labels(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_counting_detector(detector_name("Test counting labels"), "test_query", "test_object_class") iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, 0) @@ -45,9 +46,11 @@ def test_counting_labels(gl_experimental: ExperimentalApi): gl_experimental.add_label(iq1, -999) -def test_multiclass_labels(gl_experimental: ExperimentalApi): - name = f"Test binary labels{datetime.utcnow()}" - det = gl_experimental.create_multiclass_detector(name, "test_query", class_names=["apple", "banana", "cherry"]) +@retry_on_failure() +def test_multiclass_labels(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_multiclass_detector( + detector_name("Test multiclass labels"), "test_query", class_names=["apple", "banana", "cherry"] + ) iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "apple") iq1 = gl_experimental.get_image_query(iq1.id) @@ -66,9 +69,9 @@ def test_multiclass_labels(gl_experimental: ExperimentalApi): gl_experimental.add_label(iq1, "MAYBE") -def test_bounding_box_labels(gl_experimental: ExperimentalApi): - name = f"Test bounding box labels{datetime.utcnow()}" - det = gl_experimental.create_bounding_box_detector(name, "test_query", "test_class") +@retry_on_failure() +def test_bounding_box_labels(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_bounding_box_detector(detector_name("Test bounding box labels"), "test_query", "test_class") iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "NO_OBJECTS") iq1 = gl_experimental.get_image_query(iq1.id) @@ -84,9 +87,9 @@ def test_bounding_box_labels(gl_experimental: ExperimentalApi): gl_experimental.add_label(iq1, "MAYBE") -def test_text_recognition_labels(gl_experimental: ExperimentalApi): - name = f"Test text recognition labels{datetime.utcnow()}" - det = gl_experimental.create_text_recognition_detector(name, "test_query") +@retry_on_failure() +def test_text_recognition_labels(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_text_recognition_detector(detector_name("Test text recognition labels"), "test_query") iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "apple text") iq1 = gl_experimental.get_image_query(iq1.id) diff --git a/test/unit/test_metrics_and_evaluation.py b/test/unit/test_metrics_and_evaluation.py index 99924aa7..0299281f 100644 --- a/test/unit/test_metrics_and_evaluation.py +++ b/test/unit/test_metrics_and_evaluation.py @@ -1,14 +1,13 @@ import time -from datetime import datetime +from typing import Callable import pytest from groundlight import ExperimentalApi @pytest.mark.skip(reason="Slow") -def test_metrics_and_evaluation(gl_experimental: ExperimentalApi): - name = f"Test metrics and evaluation {datetime.utcnow()}" - det = gl_experimental.create_detector(name, "test_query") +def test_metrics_and_evaluation(gl_experimental: ExperimentalApi, detector_name: Callable): + det = gl_experimental.create_detector(detector_name("Test metrics and evaluation"), "test_query") for i in range(10): iq = gl_experimental.submit_image_query( det, "test/assets/cat.jpeg", wait=0, patience_time=10, human_review="NEVER" From 2dc545b3b544fd19ceede770bb3f005dcb3f47e2 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Tue, 7 Apr 2026 18:54:12 +0000 Subject: [PATCH 2/6] Automatically reformatting code --- test/integration/test_groundlight.py | 3 ++- test/unit/test_experimental.py | 1 + test/unit/test_images.py | 1 + test/unit/test_labels.py | 9 +++++++-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index c7671e3a..a0d56d83 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -26,10 +26,11 @@ PaginatedDetectorList, PaginatedImageQueryList, ) -from test.retry_decorator import retry_on_failure from urllib3.exceptions import ConnectTimeoutError, MaxRetryError, ReadTimeoutError from urllib3.util.retry import Retry +from test.retry_decorator import retry_on_failure + DEFAULT_CONFIDENCE_THRESHOLD = 0.9 IQ_IMPROVEMENT_THRESHOLD = 0.75 diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 81223aa1..1a3d80c1 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -4,6 +4,7 @@ import pytest from groundlight import ExperimentalApi from model import Detector, ImageQuery + from test.retry_decorator import retry_on_failure diff --git a/test/unit/test_images.py b/test/unit/test_images.py index 249cb7be..fe916511 100644 --- a/test/unit/test_images.py +++ b/test/unit/test_images.py @@ -2,6 +2,7 @@ import PIL from groundlight import ExperimentalApi + from test.retry_decorator import retry_on_failure diff --git a/test/unit/test_labels.py b/test/unit/test_labels.py index 90df9b9e..b303465f 100644 --- a/test/unit/test_labels.py +++ b/test/unit/test_labels.py @@ -2,6 +2,7 @@ import pytest from groundlight import ApiException, ExperimentalApi + from test.retry_decorator import retry_on_failure @@ -24,7 +25,9 @@ def test_binary_labels(gl_experimental: ExperimentalApi, detector_name: Callable @retry_on_failure() def test_counting_labels(gl_experimental: ExperimentalApi, detector_name: Callable): - det = gl_experimental.create_counting_detector(detector_name("Test counting labels"), "test_query", "test_object_class") + det = gl_experimental.create_counting_detector( + detector_name("Test counting labels"), "test_query", "test_object_class" + ) iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, 0) @@ -71,7 +74,9 @@ def test_multiclass_labels(gl_experimental: ExperimentalApi, detector_name: Call @retry_on_failure() def test_bounding_box_labels(gl_experimental: ExperimentalApi, detector_name: Callable): - det = gl_experimental.create_bounding_box_detector(detector_name("Test bounding box labels"), "test_query", "test_class") + det = gl_experimental.create_bounding_box_detector( + detector_name("Test bounding box labels"), "test_query", "test_class" + ) iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "NO_OBJECTS") iq1 = gl_experimental.get_image_query(iq1.id) From 7b5c44235bc8129f729e1ea873c92ba1ef1ecb16 Mon Sep 17 00:00:00 2001 From: Tim Huff Date: Tue, 7 Apr 2026 19:16:11 +0000 Subject: [PATCH 3/6] fixing some linter errors --- test/integration/test_groundlight.py | 3 ++- test/unit/test_experimental.py | 1 + test/unit/test_images.py | 1 + test/unit/test_labels.py | 9 +++++++-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index c7671e3a..a0d56d83 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -26,10 +26,11 @@ PaginatedDetectorList, PaginatedImageQueryList, ) -from test.retry_decorator import retry_on_failure from urllib3.exceptions import ConnectTimeoutError, MaxRetryError, ReadTimeoutError from urllib3.util.retry import Retry +from test.retry_decorator import retry_on_failure + DEFAULT_CONFIDENCE_THRESHOLD = 0.9 IQ_IMPROVEMENT_THRESHOLD = 0.75 diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 81223aa1..1a3d80c1 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -4,6 +4,7 @@ import pytest from groundlight import ExperimentalApi from model import Detector, ImageQuery + from test.retry_decorator import retry_on_failure diff --git a/test/unit/test_images.py b/test/unit/test_images.py index 249cb7be..fe916511 100644 --- a/test/unit/test_images.py +++ b/test/unit/test_images.py @@ -2,6 +2,7 @@ import PIL from groundlight import ExperimentalApi + from test.retry_decorator import retry_on_failure diff --git a/test/unit/test_labels.py b/test/unit/test_labels.py index 90df9b9e..b303465f 100644 --- a/test/unit/test_labels.py +++ b/test/unit/test_labels.py @@ -2,6 +2,7 @@ import pytest from groundlight import ApiException, ExperimentalApi + from test.retry_decorator import retry_on_failure @@ -24,7 +25,9 @@ def test_binary_labels(gl_experimental: ExperimentalApi, detector_name: Callable @retry_on_failure() def test_counting_labels(gl_experimental: ExperimentalApi, detector_name: Callable): - det = gl_experimental.create_counting_detector(detector_name("Test counting labels"), "test_query", "test_object_class") + det = gl_experimental.create_counting_detector( + detector_name("Test counting labels"), "test_query", "test_object_class" + ) iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, 0) @@ -71,7 +74,9 @@ def test_multiclass_labels(gl_experimental: ExperimentalApi, detector_name: Call @retry_on_failure() def test_bounding_box_labels(gl_experimental: ExperimentalApi, detector_name: Callable): - det = gl_experimental.create_bounding_box_detector(detector_name("Test bounding box labels"), "test_query", "test_class") + det = gl_experimental.create_bounding_box_detector( + detector_name("Test bounding box labels"), "test_query", "test_class" + ) iq1 = gl_experimental.submit_image_query(det, "test/assets/cat.jpeg") gl_experimental.add_label(iq1, "NO_OBJECTS") iq1 = gl_experimental.get_image_query(iq1.id) From 7824625ff9a8988990faf872541baa8e5acb8d13 Mon Sep 17 00:00:00 2001 From: Tim Huff Date: Tue, 7 Apr 2026 20:35:08 +0000 Subject: [PATCH 4/6] responding to AI PR feedback --- test/integration/test_groundlight.py | 4 +++- test/retry_decorator.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index a0d56d83..6dadf684 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -656,6 +656,7 @@ def test_list_image_queries_with_filter(gl: Groundlight, detector_name: Callable assert image_query.id in iq_ids +@retry_on_failure() def test_get_image_query(gl: Groundlight, image_query_yes: ImageQuery): _image_query = gl.get_image_query(id=image_query_yes.id) assert str(_image_query) @@ -663,12 +664,14 @@ def test_get_image_query(gl: Groundlight, image_query_yes: ImageQuery): assert is_valid_display_result(_image_query.result) +@retry_on_failure() def test_get_image_query_label_yes(gl: Groundlight, image_query_yes: ImageQuery): gl.add_label(image_query_yes, Label.YES) retrieved_iq = gl.get_image_query(id=image_query_yes.id) assert retrieved_iq.result.label == Label.YES +@retry_on_failure() def test_get_image_query_label_no(gl: Groundlight, image_query_no: ImageQuery): gl.add_label(image_query_no, Label.NO) retrieved_iq = gl.get_image_query(id=image_query_no.id) @@ -839,7 +842,6 @@ def test_update_detector_confidence_threshold_failure(gl: Groundlight, detector: @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.") -@retry_on_failure() def test_submit_image_query_with_inspection_id_metadata_and_want_async(gl: Groundlight, detector: Detector, image: str): inspection_id = gl.start_inspection() metadata = {"key": "value"} diff --git a/test/retry_decorator.py b/test/retry_decorator.py index a86b353a..04064478 100644 --- a/test/retry_decorator.py +++ b/test/retry_decorator.py @@ -1,13 +1,15 @@ """Test-only helpers for retrying tests affected by transient cloud timing.""" import functools +import time from typing import Any, Callable, Tuple, Type def retry_on_failure( *, max_attempts: int = 3, - exception_types: Tuple[Type[BaseException], ...] = (Exception,), + exception_types: Tuple[Type[BaseException], ...] = (AssertionError,), + retry_delay_seconds: float = 5.0, ) -> Callable[[Callable[..., Any]], Callable[..., None]]: """Run the wrapped test up to `max_attempts` times when it raises a listed exception.""" @@ -24,6 +26,7 @@ def wrapper(*args: object, **kwargs: object) -> None: except exception_types: if attempt == max_attempts - 1: raise + time.sleep(retry_delay_seconds) return wrapper From 37954c3fb9e2b11b46907b9859aa220730c421dc Mon Sep 17 00:00:00 2001 From: Tim Huff Date: Tue, 7 Apr 2026 20:37:32 +0000 Subject: [PATCH 5/6] decreasing max_attempts --- test/retry_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/retry_decorator.py b/test/retry_decorator.py index 04064478..421d1291 100644 --- a/test/retry_decorator.py +++ b/test/retry_decorator.py @@ -7,7 +7,7 @@ def retry_on_failure( *, - max_attempts: int = 3, + max_attempts: int = 2, exception_types: Tuple[Type[BaseException], ...] = (AssertionError,), retry_delay_seconds: float = 5.0, ) -> Callable[[Callable[..., Any]], Callable[..., None]]: From 74af6e329f4e49be58722d8f948180a043a58cb7 Mon Sep 17 00:00:00 2001 From: Tim Huff Date: Tue, 7 Apr 2026 21:40:43 +0000 Subject: [PATCH 6/6] decorating a few more functions --- test/integration/test_groundlight.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 6dadf684..7db79be3 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -793,6 +793,7 @@ def test_update_inspection_metadata_invalid_inspection_id(gl: Groundlight): @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint doesn't support inspection_id") +@retry_on_failure() def test_stop_inspection_pass(gl: Groundlight, detector: Detector): """Starts an inspection, submits a query with the inspection ID that should pass, stops the inspection, checks the result. @@ -842,6 +843,7 @@ def test_update_detector_confidence_threshold_failure(gl: Groundlight, detector: @pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing detector metadata.") +@retry_on_failure() def test_submit_image_query_with_inspection_id_metadata_and_want_async(gl: Groundlight, detector: Detector, image: str): inspection_id = gl.start_inspection() metadata = {"key": "value"}