FFE: add delivery delay test proving Agent bypass gap

tests/ffe/test_dynamic_evaluation.py

@@ -1,6 +1,7 @@
 """Test feature flags dynamic evaluation via Remote Config."""
 
 import json
+import time
 from http import HTTPStatus
 
 from utils import (
@@ -10,7 +11,8 @@
     features,
     remote_config as rc,
 )
-from utils.proxy.mocked_response import StaticJsonMockedTracerResponse
+from utils.proxy.mocked_response import StaticJsonMockedTracerResponse, MockedBackendResponse
+from utils.proxy.rc_response_builder import build_rc_configurations_protobuf
 
 
 RC_PRODUCT = "FFE_FLAGS"
@@ -197,3 +199,65 @@ def test_ffe_rc_down_from_start(self):
         assert result["value"] == self.default_value, (
             f"Expected default '{self.default_value}', got '{result['value']}'"
         )
+
+
+@scenarios.feature_flagging_and_experimentation_backend
+@features.feature_flags_dynamic_evaluation
+class Test_FFE_RC_Delivery_Delay:
+    """Demonstrate that FFE_FLAGS is NOT delivered within 5s when the Agent uses a 60s refresh interval.
+
+    This test exercises the real Agent RC path (rc_backend_enabled=True) with a 60s
+    background poll interval. It posts an FFE_FLAGS config to the mocked backend, waits
+    5 seconds, and asserts the tracer has NOT received it.
+
+    Root cause: the Agent's cache bypass fires on new *clients*, not new *products*.
+    The tracer's RC client is already active (started for APM), so when FFE_FLAGS is
+    registered, no bypass fires. The Agent only discovers FFE_FLAGS on its next
+    background poll — 60s later.
+
+    This test codifies the problem described in the fast-lane proposal. When the Agent
+    ships the product-aware bypass fix, this test should be updated to assert that
+    FFE_FLAGS IS delivered within 5s.
+    """
+
+    WAIT_SECONDS = 5
+
+    def setup_ffe_rc_delivery_delay(self):
+        """Post FFE_FLAGS config to mocked backend and wait 5 seconds."""
+        # Build the RC payload with FFE_FLAGS config
+        rc_state = rc.backend_rc_state
+        rc_state.reset()
+        rc_state.set_config(f"{RC_PATH}/ffe-speed-test/config", UFC_FIXTURE_DATA)
+        rc_state.version += 1
+        payload = rc_state.to_payload()
+
+        # Post protobuf to the mocked backend (Agent will pick it up on next poll)
+        rc_protobuf = build_rc_configurations_protobuf(payload)
+        MockedBackendResponse(path="/api/v0.1/configurations", content=rc_protobuf).send()
+
+        # Wait 5 seconds — enough for the tracer to poll many times (1s interval),
+        # but NOT enough for the Agent's 60s background poll to fire
+        time.sleep(self.WAIT_SECONDS)
+
+    def test_ffe_rc_delivery_delay(self):
+        """Assert FFE_FLAGS has NOT been acknowledged by the tracer within 5 seconds.
+
+        With a 60s Agent refresh interval and no product-aware bypass, the Agent's
+        cache does not contain FFE_FLAGS yet. The tracer polls the Agent every 1s
+        but gets empty responses. No FFE_FLAGS config_state should appear.
+        """
+        ffe_states_found = []
+
+        for data in interfaces.library.get_data(path_filters="/v0.7/config"):
+            config_states = (
+                data.get("request", {}).get("content", {}).get("client", {}).get("state", {}).get("config_states", [])
+            )
+            for config_state in config_states:
+                if config_state.get("product") == RC_PRODUCT:
+                    ffe_states_found.append(config_state)
+
+        assert len(ffe_states_found) == 0, (
+            f"FFE_FLAGS was delivered within {self.WAIT_SECONDS}s despite 60s Agent refresh interval. "
+            f"This should not happen without the product-aware bypass fix. "
+            f"Found {len(ffe_states_found)} FFE_FLAGS config states: {ffe_states_found}"
+        )

utils/_context/_scenarios/__init__.py

@@ -540,6 +540,21 @@ class _Scenarios:
         scenario_groups=[scenario_groups.ffe],
     )
 
+    feature_flagging_and_experimentation_backend = EndToEndScenario(
+        "FEATURE_FLAGGING_AND_EXPERIMENTATION_BACKEND",
+        rc_api_enabled=True,
+        rc_backend_enabled=True,
+        weblog_env={
+            "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true",
+            "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "1",
+        },
+        agent_env={
+            "DD_REMOTE_CONFIGURATION_REFRESH_INTERVAL": "60s",
+        },
+        doc="FFE with real Agent RC path and 60s refresh — demonstrates FFE_FLAGS is not delivered quickly without product-aware bypass",
+        scenario_groups=[scenario_groups.ffe],
+    )
+
     remote_config_mocked_backend_asm_features_nocache = EndToEndScenario(
         "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE",
         rc_api_enabled=True,

utils/_context/containers.py

@@ -754,7 +754,7 @@ def __init__(
         if rc_backend_enabled:
             tuf_root_json = get_tuf_root_json()
             environment["DD_REMOTE_CONFIGURATION_ENABLED"] = "true"
-            environment["DD_REMOTE_CONFIGURATION_REFRESH_INTERVAL"] = "5s"
+            environment.setdefault("DD_REMOTE_CONFIGURATION_REFRESH_INTERVAL", "5s")
             environment["DD_REMOTE_CONFIGURATION_NO_TLS"] = "true"
             environment["DD_REMOTE_CONFIGURATION_CONFIG_ROOT"] = tuf_root_json
             environment["DD_REMOTE_CONFIGURATION_DIRECTOR_ROOT"] = tuf_root_json