[fix] make GCP client pagination iterative

plugins/gcp/fix_plugin_gcp/gcp_client.py

@@ -159,8 +159,7 @@ def list(self, api_spec: GcpApiSpec, **kwargs: Any) -> List[Json]:
         params = {k: v.format_map(params_map) for k, v in api_spec.request_parameter.items()}
         result: List[Json] = []
 
-        def next_responses(request: Any) -> None:
-            response = request.execute()
+        def add_page(response: Json) -> None:
             page = value_in_path(response, api_spec.response_path)
             if (sub_path := api_spec.response_regional_sub_path) is not None and isinstance(page, dict):
                 for zonal_marker, zonal_response in page.items():
@@ -176,14 +175,16 @@ def next_responses(request: Any) -> None:
             else:
                 raise ValueError(f"Unexpected response type: {type(page)}")
 
-            if hasattr(executor, api_spec.next_action) and (
-                nxt_req := getattr(executor, api_spec.next_action)(previous_request=request, previous_response=response)
-            ):
-                return next_responses(nxt_req)
-
         if api_spec.service_with_region_prefix and isinstance(executor._baseUrl, str):
             executor._baseUrl = executor._baseUrl.replace(api_spec.service, f"{self.region}-{api_spec.service}", 1)
-        next_responses(getattr(executor, api_spec.action)(**params))
+        request = getattr(executor, api_spec.action)(**params)
+        while request is not None:
+            response = request.execute()
+            add_page(response)
+            if hasattr(executor, api_spec.next_action):
+                request = getattr(executor, api_spec.next_action)(previous_request=request, previous_response=response)
+            else:
+                request = None
         return result
 
     @staticmethod

plugins/gcp/test/test_gcp_client.py

@@ -0,0 +1,64 @@
+from typing import Any, Optional
+
+from google.auth.credentials import AnonymousCredentials
+
+from fix_plugin_gcp.gcp_client import GcpApiSpec, GcpClient
+
+
+class FakeRequest:
+    def __init__(self, page_number: int) -> None:
+        self.page_number = page_number
+
+    def execute(self) -> dict[str, list[dict[str, int]]]:
+        return {"items": [{"page": self.page_number}]}
+
+
+class FakeFindingsExecutor:
+    _baseUrl = "https://securitycenter.googleapis.com/"
+
+    def __init__(self, pages: int) -> None:
+        self.pages = pages
+
+    def projects(self) -> "FakeFindingsExecutor":
+        return self
+
+    def sources(self) -> "FakeFindingsExecutor":
+        return self
+
+    def findings(self) -> "FakeFindingsExecutor":
+        return self
+
+    def list(self, **_: Any) -> FakeRequest:
+        return FakeRequest(0)
+
+    def list_next(self, previous_request: FakeRequest, previous_response: Any) -> Optional[FakeRequest]:
+        del previous_response
+        next_page = previous_request.page_number + 1
+        if next_page < self.pages:
+            return FakeRequest(next_page)
+        return None
+
+
+def test_gcp_client_list_handles_many_pages(monkeypatch: Any) -> None:
+    def fake_discovery(service: str, version: str, credentials: Any, cache: Any) -> FakeFindingsExecutor:
+        del service, version, credentials, cache
+        return FakeFindingsExecutor(1500)
+
+    monkeypatch.setattr("fix_plugin_gcp.gcp_client._discovery_function", fake_discovery)
+
+    api_spec = GcpApiSpec(
+        service="securitycenter",
+        version="v1",
+        accessors=["projects", "sources", "findings"],
+        action="list",
+        request_parameter={"parent": "projects/{project}/sources/-"},
+        request_parameter_in={"project"},
+        response_path="items",
+    )
+
+    client = GcpClient(AnonymousCredentials(), project_id="test-project")  # type: ignore[arg-type]
+    result = client.list(api_spec)
+
+    assert len(result) == 1500
+    assert result[0]["page"] == 0
+    assert result[-1]["page"] == 1499