test(ffe): add agentless end-to-end harness

.github/workflows/run-end-to-end.yml

@@ -320,6 +320,9 @@ jobs:
     - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario
       if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"')
       run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION
+    - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario
+      if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"')
+      run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS
     - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario
       if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"')
       run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS

manifests/cpp_httpd.yml

@@ -39,6 +39,7 @@ manifest:
   tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet)
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented)
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented)
+  tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
   tests/ffe/test_exposures.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)

manifests/cpp_nginx.yml

@@ -270,6 +270,7 @@ manifest:
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented)
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented)
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature
+  tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
   tests/ffe/test_exposures.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)

manifests/dotnet.yml

@@ -721,6 +721,7 @@ manifest:
     - declaration: missing_feature (Not implemented yet)
       component_version: <3.22.0
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: v3.36.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184)

manifests/golang.yml

@@ -1120,6 +1120,7 @@ manifest:
   tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: missing_feature (extended event schema not yet shipped)
   tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0

manifests/java.yml

@@ -3262,6 +3262,10 @@ manifest:
       component_version: <1.52.0
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: 1.61.0-SNAPSHOT  # Normalization of telemetry keys updated in https://github.com/DataDog/dd-trace-java/pull/10823
   tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme)
+  tests/ffe/test_agentless_configuration.py:
+    - weblog_declaration:
+        "*": irrelevant
+        spring-boot: missing_feature (FFL-2693 tracks Java agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py:
     - weblog_declaration:
         "*": irrelevant

manifests/nodejs.yml

@@ -1708,6 +1708,7 @@ manifest:
     - declaration: missing_feature (Not implemented yet)
       component_version: <5.66.0
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2697 tracks Node.js agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py:
     - weblog_declaration:
         "*": incomplete_test_app

manifests/php.yml

@@ -740,6 +740,7 @@ manifest:
       component_version: <1.12.0
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3
   tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme)
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py:
     - weblog_declaration:
         "*": v1.21.0-dev

manifests/python.yml

@@ -1346,6 +1346,7 @@ manifest:
   tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash:
     - declaration: bug (INPLAT-603)
       component_version: '>=3.0.0-dev'
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2699 tracks Python agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622)
   tests/ffe/test_exposures.py: v4.2.0-dev

manifests/ruby.yml

@@ -1940,6 +1940,7 @@ manifest:
     - declaration: missing_feature (Not implemented)
       component_version: <2.7.0
   tests/docker_ssi/test_docker_ssi_servicenaming.py::TestDockerServiceNaming::test_service_name: missing_feature (No implemented)
+  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py:
     - weblog_declaration:
         "*": irrelevant

manifests/rust.yml

@@ -23,6 +23,7 @@ manifest:
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented)
   tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented)
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature
+  tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract)
   tests/ffe/test_dynamic_evaluation.py: missing_feature
   tests/ffe/test_exposures.py: missing_feature
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)

tests/ffe/test_agentless_configuration.py

@@ -0,0 +1,32 @@
+"""Test default agentless UFC delivery before FFE evaluation."""
+
+import json
+
+from utils import features, scenarios, weblog
+from utils.mocked_backend.ffe import CONFIG_PATH
+
+
+@scenarios.feature_flagging_and_experimentation_agentless
+@features.feature_flags_agentless
+class Test_FFE_Agentless_Configuration:
+    def setup_default_agentless_source(self) -> None:
+        self.response = weblog.post(
+            "/ffe",
+            json={
+                "flag": "empty-targeting-key-flag",
+                "variationType": "STRING",
+                "defaultValue": "default",
+                "targetingKey": "user-1",
+                "attributes": {},
+            },
+        )
+
+    def test_default_agentless_source(self) -> None:
+        assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}"
+        assert json.loads(self.response.text)["value"] == "on-value"
+
+        backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status()
+        assert backend_status is not None
+        assert backend_status["requests_total"] >= 1
+        assert backend_status["last_path"] == CONFIG_PATH
+        assert backend_status["last_auth_present"] is True

tests/parametric/test_ffe/test_configuration_sources.py

@@ -24,14 +24,14 @@
 from utils import features, scenarios
 from utils.dd_constants import RemoteConfigApplyState
 from utils.docker_fixtures import TestAgentAPI
-from utils.docker_fixtures._mock_ffe_agentless_backend import (
+from utils.mocked_backend.ffe import (
     CONFIG_PATH,
     MockFFEAgentlessBackendServer,
     MockFFEAgentlessBackendStatus,
 )
 
 parametrize = pytest.mark.parametrize
-pytest_plugins = ["utils.docker_fixtures._mock_ffe_agentless_backend"]
+pytest_plugins = ["utils.mocked_backend.ffe"]
 
 UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json"
 RC_PRODUCT = "FFE_FLAGS"

tests/test_the_test/scenarios.json

@@ -3348,6 +3348,9 @@
   "tests/external_processing/test_apm.py::Test_ExternalProcessing_Tracing::test_correct_span_structure": [
     "EXTERNAL_PROCESSING"
   ],
+  "tests/ffe/test_agentless_configuration.py::Test_FFE_Agentless_Configuration::test_default_agentless_source": [
+    "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"
+  ],
   "tests/integrations/test_cassandra.py::Test_Cassandra::test_main": [
     "INTEGRATIONS"
   ],
@@ -5917,4 +5920,4 @@
   "tests/test_the_test/test_version.py::test_php_version": [
     "TEST_THE_TEST"
   ]
-}
+}

tests/test_the_test/test_compute_libraries_and_scenarios.py

@@ -456,6 +456,42 @@ def test_otel_test_file(self):
             "",
         )
 
+    def test_agentless_ffe_test_file(self):
+        inputs = build_inputs(modified_files=["tests/ffe/test_agentless_configuration.py"])
+        assert_github_processor(
+            inputs,
+            default_libs_with_prod,
+            default_libs_with_dev,
+            3600,
+            "false",
+            "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS",
+            "",
+        )
+
+    def test_agentless_ffe_mocked_backend_file(self):
+        inputs = build_inputs(modified_files=["utils/mocked_backend/ffe.py"])
+        assert_github_processor(
+            inputs,
+            default_libs_with_prod,
+            default_libs_with_dev,
+            3600,
+            "false",
+            "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS,PARAMETRIC",
+            "",
+        )
+
+    def test_end_to_end_scenario_framework_file(self):
+        inputs = build_inputs(modified_files=["utils/_context/_scenarios/endtoend.py"])
+        assert_github_processor(
+            inputs,
+            default_libs_with_prod,
+            default_libs_with_dev,
+            3600,
+            "false",
+            "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS",
+            "end_to_end",
+        )
+
     def test_json_modification(self):
         inputs = build_inputs(modified_files=["tests/debugger/utils/probe_snapshot_log_line.json"])
 

tests/test_the_test/test_group_rules.py

@@ -17,9 +17,18 @@ def test_appsec():
 def test_tracer_release():
     # make an exclusion list
 
+    dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless
+    assert dormant_agentless_scenario.include_agent is False
+    assert dormant_agentless_scenario.use_proxy is False
+    assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups
+    assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups
+    assert scenario_groups.end_to_end not in dormant_agentless_scenario.scenario_groups
+    assert scenario_groups.tracer_release not in dormant_agentless_scenario.scenario_groups
+
     not_in_tracer_release_group = [
         # list of scenario that will never be part of tracer release
         scenarios.fuzzer,
+        dormant_agentless_scenario,
         scenarios.mock_the_test,
         scenarios.mock_the_test_2,
         scenarios.test_the_test,

tests/test_the_test/test_mock_ffe_agentless_backend.py

@@ -1,22 +1,25 @@
 """Unit coverage for the mock FFE agentless backend test fixture."""
 
+from unittest.mock import MagicMock
+
 import requests
 import pytest
 
-from utils import features, scenarios
+from utils import scenarios
+from utils._context._scenarios import endtoend as endtoend_scenarios
 from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment
-from utils.docker_fixtures._mock_ffe_agentless_backend import (
+from utils.mocked_backend.ffe import (
     CONFIG_PATH,
     CONFIG_QUERY,
     EXPECTED_API_KEY,
     EXPECTED_DD_ENV,
     MockFFEAgentlessBackendServer,
     UFC_RESPONSE_TYPE,
 )
+from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario
 
 
 @scenarios.test_the_test
-@features.not_reported
 def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id: str) -> None:
     server = MockFFEAgentlessBackendServer(worker_id, port=0)
     try:
@@ -50,7 +53,6 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id
 
 
 @scenarios.test_the_test
-@features.not_reported
 def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.MonkeyPatch, worker_id: str) -> None:
     monkeypatch.delenv("SYSTEM_TESTS_MOCK_FFE_AGENTLESS_BACKEND_BASE_URL", raising=False)
     monkeypatch.delenv("SYSTEM_TESTS_MOCK_AGENTLESS_BACKEND_BASE_URL", raising=False)
@@ -67,7 +69,6 @@ def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.Mon
 
 
 @scenarios.test_the_test
-@features.not_reported
 def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> None:
     server = MockFFEAgentlessBackendServer(worker_id, port=0)
     try:
@@ -87,3 +88,60 @@ def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> N
         assert "body" not in status
     finally:
         server.close()
+
+
+@scenarios.test_the_test
+def test_agentless_end_to_end_scenario_starts_backend_before_weblog() -> None:
+    scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_E2E", doc="test")
+
+    try:
+        assert scenario.agent_container not in scenario._containers  # noqa: SLF001 - focused topology test
+        scenario._start_mock_backend()  # noqa: SLF001 - focused lifecycle test
+
+        environment = scenario.weblog_infra.library_container.environment
+        assert "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE" not in environment
+        assert "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" not in environment
+        base_url = environment["DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"]
+        assert isinstance(base_url, str)
+        assert base_url.endswith(f"{CONFIG_PATH}?{CONFIG_QUERY}")
+        assert scenario.weblog_infra.library_container.extra_hosts == HOST_GATEWAY_EXTRA_HOSTS
+
+        status = scenario.mock_backend_status()
+        assert status is not None
+        assert status["requests_total"] == 0
+    finally:
+        scenario._stop_mock_backend()  # noqa: SLF001 - focused lifecycle test
+
+
+@scenarios.test_the_test
+def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STARTUP_FAILURE", doc="test")
+    backend = MagicMock(spec=MockFFEAgentlessBackendServer)
+    backend.reset.side_effect = RuntimeError("reset failed")
+
+    def create_backend() -> MagicMock:
+        return backend
+
+    monkeypatch.setattr(endtoend_scenarios, "MockFFEAgentlessBackendServer", create_backend)
+
+    with pytest.raises(RuntimeError, match="reset failed"):
+        scenario.configure(MagicMock(spec=pytest.Config))
+
+    backend.close.assert_called_once_with()
+    assert scenario._mock_backend is None  # noqa: SLF001 - focused lifecycle test
+
+
+@scenarios.test_the_test
+def test_agentless_end_to_end_scenario_closes_backend_when_status_fails() -> None:
+    scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STATUS_FAILURE", doc="test")
+    backend = MagicMock(spec=MockFFEAgentlessBackendServer)
+    backend.status.side_effect = RuntimeError("status failed")
+    scenario._mock_backend = backend  # noqa: SLF001 - focused lifecycle test
+
+    with pytest.raises(RuntimeError, match="status failed"):
+        scenario._stop_mock_backend()  # noqa: SLF001 - focused lifecycle test
+
+    backend.close.assert_called_once_with()
+    assert scenario._mock_backend is None  # noqa: SLF001 - focused lifecycle test

utils/_context/_scenarios/__init__.py

@@ -7,7 +7,12 @@
 from .aws_lambda import LambdaScenario
 from .core import Scenario, scenario_groups
 from .default import DefaultScenario
-from .endtoend import DockerScenario, DdTraceEndToEndScenario, GraphQlEndToEndScenario
+from .endtoend import (
+    DockerScenario,
+    DdTraceEndToEndScenario,
+    FeatureFlaggingAgentlessEndToEndScenario,
+    GraphQlEndToEndScenario,
+)
 from .integrations import (
     CrossedTracingLibraryScenario,
     DbmDynamicServiceScenario,
@@ -727,6 +732,10 @@ class _Scenarios:
         scenario_groups=[scenario_groups.ffe],
     )
 
+    feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario(
+        "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"
+    )
+
     remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario(
         "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE",
         rc_api_enabled=True,