From 8c6712f719fd4b2c98c166130881ec0372e42996 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 20:38:42 -0400 Subject: [PATCH 1/9] Pin FFE end-to-end scenario to remote config --- utils/_context/_scenarios/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index c93312a3e67..15a9c0f2b77 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -704,6 +704,7 @@ class _Scenarios: rc_api_enabled=True, weblog_env={ "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", # set_provider() in Python blocks until we receive RC # configuration for feature flags. But it is only sent # after weblog sucessfully boots and tests start From cf6126acdf432db3101a12a37c2e957e22ce95e6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 22:00:54 -0400 Subject: [PATCH 2/9] test(ffe): add agentless end-to-end harness --- .github/workflows/run-end-to-end.yml | 3 + manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + tests/ffe/test_agentless_configuration.py | 32 +++++++++++ .../test_mock_ffe_agentless_backend.py | 23 ++++++++ utils/_context/_scenarios/__init__.py | 17 +++++- utils/_context/_scenarios/endtoend.py | 56 +++++++++++++++++++ utils/_context/containers.py | 5 ++ utils/docker_fixtures/_core.py | 4 +- 14 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 tests/ffe/test_agentless_configuration.py diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 9415bc5a3cd..a830e1d1c4b 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -320,6 +320,9 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION + - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') + run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 5d7d7c329a6..8ff3aa9bfc4 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -718,6 +718,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <3.22.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) diff --git a/manifests/golang.yml b/manifests/golang.yml index d5ffcf8aaa3..89a68ba29e0 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1117,6 +1117,7 @@ manifest: tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: missing_feature (extended event schema not yet shipped) tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0 + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 diff --git a/manifests/java.yml b/manifests/java.yml index 39b7d220e43..209df5639a6 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3249,6 +3249,7 @@ manifest: component_version: <1.52.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: 1.61.0-SNAPSHOT # Normalization of telemetry keys updated in https://github.com/DataDog/dd-trace-java/pull/10823 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index e66e0bf8384..05eb5f818ff 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1705,6 +1705,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/php.yml b/manifests/php.yml index 16689713855..26e0dcfe281 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -709,6 +709,7 @@ manifest: component_version: <1.12.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index 8db5c9a51ce..8b1bf2060cd 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1343,6 +1343,7 @@ manifest: tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 97c03c964f6..0559dfe53fb 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1929,6 +1929,7 @@ manifest: - declaration: missing_feature (Not implemented) component_version: <2.7.0 tests/docker_ssi/test_docker_ssi_servicenaming.py::TestDockerServiceNaming::test_service_name: missing_feature (No implemented) + tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py new file mode 100644 index 00000000000..2e5ef64447b --- /dev/null +++ b/tests/ffe/test_agentless_configuration.py @@ -0,0 +1,32 @@ +"""Test default agentless UFC delivery before FFE evaluation.""" + +import json + +from utils import features, scenarios, weblog +from utils.docker_fixtures._mock_ffe_agentless_backend import CONFIG_PATH + + +@scenarios.feature_flagging_and_experimentation_agentless +@features.feature_flags_agentless +class Test_FFE_Agentless_Configuration: + def setup_default_agentless_source(self) -> None: + self.response = weblog.post( + "/ffe", + json={ + "flag": "empty-targeting-key-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + self.backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() + + def test_default_agentless_source(self) -> None: + assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" + assert json.loads(self.response.text)["value"] == "on-value" + + assert self.backend_status is not None + assert self.backend_status["requests_total"] >= 1 + assert self.backend_status["last_path"] == CONFIG_PATH + assert self.backend_status["last_auth_present"] is True diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index b192eb72dc0..bb559292662 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -10,6 +10,7 @@ EXPECTED_API_KEY, MockFFEAgentlessBackendServer, ) +from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario @scenarios.test_the_test @@ -66,3 +67,25 @@ def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> N assert "body" not in status finally: server.close() + + +@scenarios.test_the_test +@features.not_reported +def test_agentless_end_to_end_scenario_starts_backend_before_weblog(worker_id: str) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_E2E", doc="test") + + try: + scenario._start_mock_backend(worker_id) # noqa: SLF001 - focused lifecycle test + + environment = scenario.weblog_infra.library_container.environment + assert "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE" not in environment + base_url = environment["DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"] + assert isinstance(base_url, str) + assert base_url.endswith(CONFIG_PATH) + assert scenario.weblog_infra.library_container.extra_hosts == HOST_GATEWAY_EXTRA_HOSTS + + status = scenario.mock_backend_status() + assert status is not None + assert status["requests_total"] == 0 + finally: + scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 15a9c0f2b77..c6c9abf87f7 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -7,7 +7,7 @@ from .aws_lambda import LambdaScenario from .core import Scenario, scenario_groups from .default import DefaultScenario -from .endtoend import DockerScenario, EndToEndScenario +from .endtoend import DockerScenario, EndToEndScenario, FeatureFlaggingAgentlessEndToEndScenario from .integrations import ( CrossedTracingLibraryScenario, DbmDynamicServiceScenario, @@ -724,6 +724,21 @@ class _Scenarios: scenario_groups=[scenario_groups.ffe], ) + feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + weblog_env={ + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_METRICS_OTEL_ENABLED": "true", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", + "OTEL_METRIC_EXPORT_INTERVAL": "1000", + }, + doc="Validate default agentless UFC delivery and FFE side effects through the Agent telemetry conduit.", + scenario_groups=[scenario_groups.ffe], + ) + remote_config_mocked_backend_asm_features_nocache = EndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 4c3fb3d2a23..9e89b06af1b 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -21,6 +21,12 @@ TestedContainer, ) from utils._context.weblog_infrastructure import EndToEndWeblogInfra +from utils.docker_fixtures._core import extra_hosts_for_environment +from utils.docker_fixtures._mock_ffe_agentless_backend import ( + EXPECTED_API_KEY, + MockFFEAgentlessBackendServer, + MockFFEAgentlessBackendStatus, +) from utils._logger import logger @@ -517,3 +523,53 @@ def get_junit_properties(self) -> dict[str, str]: @property def appsec_rules_file(self) -> str | None: return self.weblog_infra.appsec_rules_file + + +class FeatureFlaggingAgentlessEndToEndScenario(EndToEndScenario): + """FFE end-to-end scenario with UFC available before the weblog starts.""" + + _mock_backend: MockFFEAgentlessBackendServer | None = None + _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None + + def configure(self, config: pytest.Config) -> None: + if not self.replay: + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + self._start_mock_backend(worker_id) + + try: + super().configure(config) + except BaseException: + self._stop_mock_backend() + raise + + def _start_mock_backend(self, worker_id: str) -> None: + assert self._mock_backend is None, "mock FFE agentless backend is already running" + + self._mock_backend = MockFFEAgentlessBackendServer(worker_id) + self._mock_backend.reset() + + environment = self.weblog_infra.library_container.environment + environment |= { + "DD_API_KEY": EXPECTED_API_KEY, + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": self._mock_backend.library_config_url, + } + self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + + def mock_backend_status(self) -> MockFFEAgentlessBackendStatus | None: + if self._mock_backend is not None: + return self._mock_backend.status() + return self._last_mock_backend_status + + def _stop_mock_backend(self) -> None: + if self._mock_backend is None: + return + + self._last_mock_backend_status = self._mock_backend.status() + self._mock_backend.close() + self._mock_backend = None + + def close_targets(self) -> None: + try: + super().close_targets() + finally: + self._stop_mock_backend() diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 59317347c56..8376d418eda 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -22,6 +22,7 @@ from utils._context.docker import get_docker_client from utils._context._image_mirror import mirror_image from utils._context.constants import ContainerPorts +from utils.docker_fixtures._core import extra_hosts_for_environment from utils.proxy.tuf import get_tuf_root_json from utils.proxy.ports import ProxyPorts from utils.proxy.mocked_response import ( @@ -86,6 +87,7 @@ def __init__( cap_add: list[str] | None = None, command: str | list[str] | None = None, environment: dict[str, str | None] | None = None, + extra_hosts: dict[str, str] | None = None, healthcheck: dict | None = None, local_image_only: bool = False, ports: dict | None = None, @@ -115,6 +117,7 @@ def __init__( self.healthy: bool | None = None self.environment = environment or {} + self.extra_hosts = extra_hosts self.volumes = volumes or {} self.ports = ports or {} self.depends_on: list[TestedContainer] = [] @@ -229,6 +232,7 @@ def start(self, network: Network) -> Container: name=self.container_name, hostname=self.name, environment=self.environment, + extra_hosts=self.extra_hosts, # auto_remove=True, detach=True, network=network.name, @@ -949,6 +953,7 @@ def __init__( image_name="system_tests/weblog", name="weblog", environment=environment, + extra_hosts=extra_hosts_for_environment(environment), volumes=volumes, # ddprof's perf event open is blocked by default by docker's seccomp profile # This is worse than the line above though prevents mmap bugs locally diff --git a/utils/docker_fixtures/_core.py b/utils/docker_fixtures/_core.py index 3cfacedd571..ddda56f4ce0 100644 --- a/utils/docker_fixtures/_core.py +++ b/utils/docker_fixtures/_core.py @@ -1,5 +1,5 @@ import contextlib -from collections.abc import Generator +from collections.abc import Generator, Mapping from pathlib import Path from typing import TextIO from urllib.parse import urlparse @@ -44,7 +44,7 @@ def compute_volumes(volumes: dict[str, str]) -> dict[str, dict]: return fixed_volumes -def extra_hosts_for_environment(env: dict[str, str]) -> dict[str, str] | None: +def extra_hosts_for_environment(env: Mapping[str, str | None]) -> dict[str, str] | None: for value in env.values(): if urlparse(str(value)).hostname == HOST_DOCKER_INTERNAL: return dict(HOST_GATEWAY_EXTRA_HOSTS) From 5d5e341f9232ab4d4d730ad302f24ba9be4a29f3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 13 Jul 2026 21:37:50 -0400 Subject: [PATCH 3/9] test(ffe): address agentless harness review feedback --- tests/ffe/test_agentless_configuration.py | 10 +++++----- tests/test_the_test/scenarios.json | 5 ++++- .../test_compute_libraries_and_scenarios.py | 12 ++++++++++++ tests/test_the_test/test_group_rules.py | 7 +++++++ utils/_context/_scenarios/__init__.py | 1 + utils/_context/_scenarios/endtoend.py | 18 +++++++++++++----- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py index 2e5ef64447b..fdbce85d180 100644 --- a/tests/ffe/test_agentless_configuration.py +++ b/tests/ffe/test_agentless_configuration.py @@ -20,13 +20,13 @@ def setup_default_agentless_source(self) -> None: "attributes": {}, }, ) - self.backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() def test_default_agentless_source(self) -> None: assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" assert json.loads(self.response.text)["value"] == "on-value" - assert self.backend_status is not None - assert self.backend_status["requests_total"] >= 1 - assert self.backend_status["last_path"] == CONFIG_PATH - assert self.backend_status["last_auth_present"] is True + backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() + assert backend_status is not None + assert backend_status["requests_total"] >= 1 + assert backend_status["last_path"] == CONFIG_PATH + assert backend_status["last_auth_present"] is True diff --git a/tests/test_the_test/scenarios.json b/tests/test_the_test/scenarios.json index 5967b859045..43a32cc149f 100644 --- a/tests/test_the_test/scenarios.json +++ b/tests/test_the_test/scenarios.json @@ -3348,6 +3348,9 @@ "tests/external_processing/test_apm.py::Test_ExternalProcessing_Tracing::test_correct_span_structure": [ "EXTERNAL_PROCESSING" ], + "tests/ffe/test_agentless_configuration.py::Test_FFE_Agentless_Configuration::test_default_agentless_source": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" + ], "tests/integrations/test_cassandra.py::Test_Cassandra::test_main": [ "INTEGRATIONS" ], @@ -5917,4 +5920,4 @@ "tests/test_the_test/test_version.py::test_php_version": [ "TEST_THE_TEST" ] -} \ No newline at end of file +} diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index aaf6b694798..72108957cf4 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -456,6 +456,18 @@ def test_otel_test_file(self): "", ) + def test_agentless_ffe_test_file(self): + inputs = build_inputs(modified_files=["tests/ffe/test_agentless_configuration.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "", + ) + def test_json_modification(self): inputs = build_inputs(modified_files=["tests/debugger/utils/probe_snapshot_log_line.json"]) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index b636ea46a4a..8d53ecbbbc5 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -17,9 +17,16 @@ def test_appsec(): def test_tracer_release(): # make an exclusion list + dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless + assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups + assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups + assert scenario_groups.end_to_end not in dormant_agentless_scenario.scenario_groups + assert scenario_groups.tracer_release not in dormant_agentless_scenario.scenario_groups + not_in_tracer_release_group = [ # list of scenario that will never be part of tracer release scenarios.fuzzer, + dormant_agentless_scenario, scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 3ded265dc52..0a4d1ef3aed 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -739,6 +739,7 @@ class _Scenarios: "OTEL_METRIC_EXPORT_INTERVAL": "1000", }, doc="Validate default agentless UFC delivery and FFE side effects through the Agent telemetry conduit.", + include_default_scenario_groups=False, scenario_groups=[scenario_groups.ffe], ) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 10c5d24a5a0..4e7adc73740 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -223,15 +223,21 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, require_api_key: bool = False, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: - scenario_groups = [ - all_scenario_groups.all, - all_scenario_groups.end_to_end, - all_scenario_groups.tracer_release, - ] + (scenario_groups or []) + default_scenario_groups = ( + [ + all_scenario_groups.all, + all_scenario_groups.end_to_end, + all_scenario_groups.tracer_release, + ] + if include_default_scenario_groups + else [] + ) + scenario_groups = default_scenario_groups + (scenario_groups or []) super().__init__( name, @@ -548,6 +554,7 @@ def __init__( backend_interface_timeout: int = 0, client_drop_p0s: bool | None = None, iast_enabled: bool = True, + include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, library_interface_timeout: int | None = None, meta_structs_disabled: bool = False, @@ -572,6 +579,7 @@ def __init__( client_drop_p0s=client_drop_p0s, doc=doc, iast_enabled=iast_enabled, + include_default_scenario_groups=include_default_scenario_groups, include_opentelemetry=include_opentelemetry, library_interface_timeout=library_interface_timeout, meta_structs_disabled=meta_structs_disabled, From 73ad33184bf995b67bc30e5f71b30e36782a8598 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 13 Jul 2026 22:19:39 -0400 Subject: [PATCH 4/9] test(ffe): remove Agent from agentless scenario --- tests/test_the_test/test_group_rules.py | 2 + .../test_mock_ffe_agentless_backend.py | 10 +++- utils/_context/_scenarios/__init__.py | 11 ++-- utils/_context/_scenarios/endtoend.py | 59 +++++++++++++------ utils/_context/weblog_infrastructure.py | 5 +- 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index 8d53ecbbbc5..dda1b74ffbf 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -18,6 +18,8 @@ def test_tracer_release(): # make an exclusion list dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless + assert dormant_agentless_scenario.include_agent is False + assert dormant_agentless_scenario.use_proxy is False assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups assert scenario_groups.end_to_end not in dormant_agentless_scenario.scenario_groups diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index bb559292662..759bb628caa 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -72,13 +72,21 @@ def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> N @scenarios.test_the_test @features.not_reported def test_agentless_end_to_end_scenario_starts_backend_before_weblog(worker_id: str) -> None: - scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_E2E", doc="test") + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_E2E", + doc="test", + include_agent=False, + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) try: + assert scenario.agent_container not in scenario._containers # noqa: SLF001 - focused topology test scenario._start_mock_backend(worker_id) # noqa: SLF001 - focused lifecycle test environment = scenario.weblog_infra.library_container.environment assert "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE" not in environment + assert "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" not in environment base_url = environment["DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"] assert isinstance(base_url, str) assert base_url.endswith(CONFIG_PATH) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 0a4d1ef3aed..3f69b4d1bd9 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -733,14 +733,15 @@ class _Scenarios: "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", - "DD_METRICS_OTEL_ENABLED": "true", - "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", - "OTEL_METRIC_EXPORT_INTERVAL": "1000", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", }, - doc="Validate default agentless UFC delivery and FFE side effects through the Agent telemetry conduit.", + doc="Validate default agentless UFC delivery and evaluation without a Datadog Agent.", + include_agent=False, include_default_scenario_groups=False, + library_interface_timeout=0, scenario_groups=[scenario_groups.ffe], + use_proxy_for_agent=False, + use_proxy_for_weblog=False, ) remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 4e7adc73740..df66d08425a 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -12,6 +12,7 @@ from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.buddies import BuddyHostPorts from utils.proxy.ports import ProxyPorts +from utils._context.component_version import Version from utils._context.docker import get_docker_client from utils._context.containers import ( WeblogContainer, @@ -192,7 +193,7 @@ def close_targets(self): class EndToEndScenario(DockerScenario): - """Scenario that implier an instrumented HTTP application shipping a datadog tracer (weblog) and an datadog agent""" + """Scenario with an instrumented HTTP application and an optional Datadog Agent.""" def __init__( self, @@ -223,6 +224,7 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_agent: bool = True, include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, require_api_key: bool = False, @@ -246,7 +248,7 @@ def __init__( scenario_groups=scenario_groups, weblog_categories=weblog_categories, enable_ipv6=enable_ipv6, - use_proxy=use_proxy_for_agent or use_proxy_for_weblog, + use_proxy=(include_agent and use_proxy_for_agent) or use_proxy_for_weblog, rc_api_enabled=rc_api_enabled, rc_backend_enabled=rc_backend_enabled, meta_structs_disabled=meta_structs_disabled, @@ -255,14 +257,19 @@ def __init__( obfuscation_version=obfuscation_version, ) - self._use_proxy_for_agent = use_proxy_for_agent + if include_buddies and not include_agent: + raise ValueError("include_buddies requires include_agent") + + self.include_agent = include_agent + self._use_proxy_for_agent = include_agent and use_proxy_for_agent self._use_proxy_for_weblog = use_proxy_for_weblog self._require_api_key = require_api_key self.agent_container = AgentContainer( use_proxy=use_proxy_for_agent, rc_backend_enabled=rc_backend_enabled, environment=agent_env ) - self._containers.append(self.agent_container) + if include_agent: + self._containers.append(self.agent_container) self._weblog_env = dict(weblog_env) if weblog_env else {} self.weblog_infra = EndToEndWeblogInfra( @@ -327,12 +334,14 @@ def configure(self, config: pytest.Config): self._set_containers_dependancies() super().configure(config) - interfaces.agent.configure(self.host_log_folder, replay=self.replay) + if self.include_agent: + interfaces.agent.configure(self.host_log_folder, replay=self.replay) interfaces.library.configure(self.host_log_folder, replay=self.replay) interfaces.backend.configure(self.host_log_folder, replay=self.replay) interfaces.library_dotnet_managed.configure(self.host_log_folder, replay=self.replay) interfaces.library_stdout.configure(self.host_log_folder, replay=self.replay) - interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) + if self.include_agent: + interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) if self.include_opentelemetry: interfaces.open_telemetry.configure(self.host_log_folder, replay=self.replay) @@ -377,7 +386,8 @@ def _set_containers_dependancies(self) -> None: self.agent_container.depends_on.append(self.proxy_container) proxy_container = self.proxy_container if self._use_proxy_for_weblog else None - self.weblog_infra.set_weblog_dependencies(self.agent_container, proxy_container) + agent_container = self.agent_container if self.include_agent else None + self.weblog_infra.set_weblog_dependencies(agent_container, proxy_container) for buddy in self.buddies: buddy.depends_on.append(self.agent_container) @@ -386,8 +396,10 @@ def _start_interfaces_watchdog(self): open_telemetry_interfaces: list[ProxyBasedInterfaceValidator] = ( [interfaces.open_telemetry] if self.include_opentelemetry else [] ) + agent_interfaces = [interfaces.agent] if self.include_agent else [] super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [interfaces.library] + + agent_interfaces + [container.interface for container in self.buddies] + open_telemetry_interfaces ) @@ -397,7 +409,8 @@ def _set_weblog_domain(self): self.weblog_container.set_weblog_domain_for_ipv6(self._network) def _set_components(self): - self.components["agent"] = self.agent_version + if self.include_agent: + self.components["agent"] = self.agent_version self.components["library"] = self.library.version self.components[self.library.name] = self.library.version @@ -444,8 +457,9 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): container.interface.load_data_from_logs() container.interface.check_deserialization_errors() - interfaces.agent.load_data_from_logs() - interfaces.agent.check_deserialization_errors() + if self.include_agent: + interfaces.agent.load_data_from_logs() + interfaces.agent.check_deserialization_errors() interfaces.backend.load_data_from_logs() @@ -467,11 +481,12 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): container.stop() container.interface.check_deserialization_errors() - self._wait_interface( - interfaces.agent, 0 if force_interface_timout_to_zero else self.agent_interface_timeout - ) - self.agent_container.stop() - interfaces.agent.check_deserialization_errors() + if self.include_agent: + self._wait_interface( + interfaces.agent, 0 if force_interface_timout_to_zero else self.agent_interface_timeout + ) + self.agent_container.stop() + interfaces.agent.check_deserialization_errors() self._wait_interface( interfaces.backend, 0 if force_interface_timout_to_zero else self.backend_interface_timeout @@ -494,7 +509,9 @@ def weblog_container(self) -> WeblogContainer: @property def dd_site(self): - return self.agent_container.dd_site + if self.include_agent: + return self.agent_container.dd_site + return self.weblog_container.environment.get("DD_SITE", "datad0g.com") @property def library(self): @@ -502,7 +519,7 @@ def library(self): @property def agent_version(self): - return self.agent_container.agent_version + return self.agent_container.agent_version if self.include_agent else Version("0.0.0") @property def weblog_variant(self) -> str: @@ -554,6 +571,7 @@ def __init__( backend_interface_timeout: int = 0, client_drop_p0s: bool | None = None, iast_enabled: bool = True, + include_agent: bool = True, include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, library_interface_timeout: int | None = None, @@ -567,6 +585,8 @@ def __init__( scenario_groups: list[ScenarioGroup] | None = None, span_events: bool = True, tracer_sampling_rate: float | None = None, + use_proxy_for_agent: bool = True, + use_proxy_for_weblog: bool = True, weblog_env: dict[str, str | None] | None = None, weblog_volumes: dict | None = None, ) -> None: @@ -579,6 +599,7 @@ def __init__( client_drop_p0s=client_drop_p0s, doc=doc, iast_enabled=iast_enabled, + include_agent=include_agent, include_default_scenario_groups=include_default_scenario_groups, include_opentelemetry=include_opentelemetry, library_interface_timeout=library_interface_timeout, @@ -592,6 +613,8 @@ def __init__( scenario_groups=scenario_groups, span_events=span_events, tracer_sampling_rate=tracer_sampling_rate, + use_proxy_for_agent=use_proxy_for_agent, + use_proxy_for_weblog=use_proxy_for_weblog, weblog_categories=[WeblogCategory.dd_trace], weblog_env=weblog_env, weblog_volumes=weblog_volumes, diff --git a/utils/_context/weblog_infrastructure.py b/utils/_context/weblog_infrastructure.py index 0a99f700c6b..3796727dff3 100644 --- a/utils/_context/weblog_infrastructure.py +++ b/utils/_context/weblog_infrastructure.py @@ -133,13 +133,14 @@ def _configure_proxy_weblog(self, weblog: str) -> None: self._proxy_runtime_container.depends_on = [self._processor_container, self._dummy_server_container] def set_weblog_dependencies( - self, agent_container: TestedContainer, proxy_container: TestedContainer | None + self, agent_container: TestedContainer | None, proxy_container: TestedContainer | None ) -> None: """Wire container start-order dependencies for all weblog containers. Handles both the standard weblog topology and the go-proxies one transparently. """ - self.library_container.depends_on.append(agent_container) + if agent_container is not None: + self.library_container.depends_on.append(agent_container) if proxy_container is not None: self.library_container.depends_on.append(proxy_container) From 9db41fcd99161acb20fb3965643c7e3c25cfcea0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 13 Jul 2026 23:31:23 -0400 Subject: [PATCH 5/9] test(ffe): keep agentless harness dormant --- .github/workflows/run-end-to-end.yml | 3 -- tests/test_the_test/test_group_rules.py | 1 + .../test_mock_ffe_agentless_backend.py | 35 ++++++++++++++++++- utils/_context/_scenarios/__init__.py | 1 + utils/_context/_scenarios/endtoend.py | 6 +++- utils/_context/weblog_infrastructure.py | 7 ++-- 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index a830e1d1c4b..9415bc5a3cd 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -320,9 +320,6 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION - - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario - if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') - run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index dda1b74ffbf..d6b3b18ec93 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -19,6 +19,7 @@ def test_tracer_release(): dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless assert dormant_agentless_scenario.include_agent is False + assert dormant_agentless_scenario._flush_weblog_on_stop is False # noqa: SLF001 - no-Agent teardown contract assert dormant_agentless_scenario.use_proxy is False assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index 759bb628caa..9db2a91f2cd 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -3,7 +3,7 @@ import requests import pytest -from utils import features, scenarios +from utils import features, interfaces, scenarios from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment from utils.docker_fixtures._mock_ffe_agentless_backend import ( CONFIG_PATH, @@ -97,3 +97,36 @@ def test_agentless_end_to_end_scenario_starts_backend_before_weblog(worker_id: s assert status["requests_total"] == 0 finally: scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test + + +@scenarios.test_the_test +@features.not_reported +def test_agentless_end_to_end_scenario_stops_without_tracer_flush(monkeypatch: pytest.MonkeyPatch) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_NO_FLUSH", + doc="test", + flush_weblog_on_stop=False, + include_agent=False, + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + flushed = False + stopped = False + + def flush() -> None: + nonlocal flushed + flushed = True + + def stop() -> None: + nonlocal stopped + stopped = True + + monkeypatch.setattr(scenario, "_wait_interface", lambda *_: None) + monkeypatch.setattr(scenario.weblog_infra.http_container, "flush", flush) + monkeypatch.setattr(scenario.weblog_infra.http_container, "stop", stop) + monkeypatch.setattr(interfaces.library, "check_deserialization_errors", lambda: None) + + scenario._wait_and_stop_containers(force_interface_timout_to_zero=True) # noqa: SLF001 + + assert stopped is True + assert flushed is False diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 3f69b4d1bd9..1764bc43436 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -738,6 +738,7 @@ class _Scenarios: doc="Validate default agentless UFC delivery and evaluation without a Datadog Agent.", include_agent=False, include_default_scenario_groups=False, + flush_weblog_on_stop=False, library_interface_timeout=0, scenario_groups=[scenario_groups.ffe], use_proxy_for_agent=False, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index df66d08425a..0af7853291f 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -228,6 +228,7 @@ def __init__( include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, require_api_key: bool = False, + flush_weblog_on_stop: bool = True, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: default_scenario_groups = ( @@ -263,6 +264,7 @@ def __init__( self.include_agent = include_agent self._use_proxy_for_agent = include_agent and use_proxy_for_agent self._use_proxy_for_weblog = use_proxy_for_weblog + self._flush_weblog_on_stop = flush_weblog_on_stop self._require_api_key = require_api_key self.agent_container = AgentContainer( @@ -472,7 +474,7 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.library, 0 if force_interface_timout_to_zero else self.library_interface_timeout ) - self.weblog_infra.stop() + self.weblog_infra.stop(flush=self._flush_weblog_on_stop) interfaces.library.check_deserialization_errors() for container in self.buddies: @@ -581,6 +583,7 @@ def __init__( rc_api_enabled: bool = False, rc_backend_enabled: bool = False, require_api_key: bool = False, + flush_weblog_on_stop: bool = True, runtime_metrics_enabled: bool = False, scenario_groups: list[ScenarioGroup] | None = None, span_events: bool = True, @@ -609,6 +612,7 @@ def __init__( rc_api_enabled=rc_api_enabled, rc_backend_enabled=rc_backend_enabled, require_api_key=require_api_key, + flush_weblog_on_stop=flush_weblog_on_stop, runtime_metrics_enabled=runtime_metrics_enabled, scenario_groups=scenario_groups, span_events=span_events, diff --git a/utils/_context/weblog_infrastructure.py b/utils/_context/weblog_infrastructure.py index 3796727dff3..175ac6a95f5 100644 --- a/utils/_context/weblog_infrastructure.py +++ b/utils/_context/weblog_infrastructure.py @@ -33,7 +33,7 @@ def configure(self, config: pytest.Config) -> None: """Perform any configuration. Executed only if the weblog will be used""" @abstractmethod - def stop(self) -> None: + def stop(self, *, flush: bool = True) -> None: """Stop the tested infra""" @@ -208,7 +208,7 @@ def get_containers(self) -> tuple[TestedContainer, ...]: ) return (self.http_container, *self._other_containers) - def stop(self) -> None: + def stop(self, *, flush: bool = True) -> None: if self._is_proxy_weblog: if self._proxy_runtime_container: self._proxy_runtime_container.stop() @@ -217,7 +217,8 @@ def stop(self) -> None: if self._dummy_server_container: self._dummy_server_container.stop() else: - self.http_container.flush() + if flush: + self.http_container.flush() self.http_container.stop() @property From 71aacb0987630edd5198df877a67f98b11546f8c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 14 Jul 2026 08:46:55 -0400 Subject: [PATCH 6/9] test(ffe): harden dormant agentless harness --- manifests/java.yml | 5 +- tests/ffe/test_agentless_configuration.py | 10 ++-- tests/test_the_test/test_group_rules.py | 1 + .../test_mock_ffe_agentless_backend.py | 51 +++++++++++++++++++ utils/_context/_scenarios/__init__.py | 1 + utils/_context/_scenarios/endtoend.py | 21 +++++--- 6 files changed, 75 insertions(+), 14 deletions(-) diff --git a/manifests/java.yml b/manifests/java.yml index 1b7f54a25e3..629f9163c3a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3261,7 +3261,10 @@ manifest: component_version: <1.52.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: 1.61.0-SNAPSHOT # Normalization of telemetry keys updated in https://github.com/DataDog/dd-trace-java/pull/10823 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: + - weblog_declaration: + "*": irrelevant + spring-boot: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py index fdbce85d180..2e5ef64447b 100644 --- a/tests/ffe/test_agentless_configuration.py +++ b/tests/ffe/test_agentless_configuration.py @@ -20,13 +20,13 @@ def setup_default_agentless_source(self) -> None: "attributes": {}, }, ) + self.backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() def test_default_agentless_source(self) -> None: assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" assert json.loads(self.response.text)["value"] == "on-value" - backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() - assert backend_status is not None - assert backend_status["requests_total"] >= 1 - assert backend_status["last_path"] == CONFIG_PATH - assert backend_status["last_auth_present"] is True + assert self.backend_status is not None + assert self.backend_status["requests_total"] >= 1 + assert self.backend_status["last_path"] == CONFIG_PATH + assert self.backend_status["last_auth_present"] is True diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index d6b3b18ec93..083033fe03f 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -18,6 +18,7 @@ def test_tracer_release(): # make an exclusion list dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless + assert dormant_agentless_scenario.github_workflow is None assert dormant_agentless_scenario.include_agent is False assert dormant_agentless_scenario._flush_weblog_on_stop is False # noqa: SLF001 - no-Agent teardown contract assert dormant_agentless_scenario.use_proxy is False diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index 9db2a91f2cd..0511085f5a0 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -1,9 +1,12 @@ """Unit coverage for the mock FFE agentless backend test fixture.""" +from unittest.mock import MagicMock + import requests import pytest from utils import features, interfaces, scenarios +from utils._context._scenarios import endtoend as endtoend_scenarios from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment from utils.docker_fixtures._mock_ffe_agentless_backend import ( CONFIG_PATH, @@ -130,3 +133,51 @@ def stop() -> None: assert stopped is True assert flushed is False + + +@scenarios.test_the_test +@features.not_reported +def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_STARTUP_FAILURE", + doc="test", + include_agent=False, + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + backend = MagicMock(spec=MockFFEAgentlessBackendServer) + backend.reset.side_effect = RuntimeError("reset failed") + + def create_backend(_worker_id: str) -> MagicMock: + return backend + + monkeypatch.setattr(endtoend_scenarios, "MockFFEAgentlessBackendServer", create_backend) + + with pytest.raises(RuntimeError, match="reset failed"): + scenario.configure(MagicMock(spec=pytest.Config)) + + backend.close.assert_called_once_with() + assert scenario._mock_backend is None # noqa: SLF001 - focused lifecycle test + + +@scenarios.test_the_test +@features.not_reported +def test_agentless_end_to_end_scenario_closes_backend_when_status_fails() -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario( + "MOCK_FFE_AGENTLESS_STATUS_FAILURE", + doc="test", + include_agent=False, + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + ) + backend = MagicMock(spec=MockFFEAgentlessBackendServer) + backend.status.side_effect = RuntimeError("status failed") + scenario._mock_backend = backend # noqa: SLF001 - focused lifecycle test + + with pytest.raises(RuntimeError, match="status failed"): + scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test + + backend.close.assert_called_once_with() + assert scenario._mock_backend is None # noqa: SLF001 - focused lifecycle test diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 1764bc43436..3fe0f610ea4 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -739,6 +739,7 @@ class _Scenarios: include_agent=False, include_default_scenario_groups=False, flush_weblog_on_stop=False, + github_workflow=None, library_interface_timeout=0, scenario_groups=[scenario_groups.ffe], use_proxy_for_agent=False, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 0af7853291f..c679a08ce94 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -200,7 +200,7 @@ def __init__( name: str, *, doc: str, - github_workflow: str = "endtoend", + github_workflow: str | None = "endtoend", weblog_categories: list[WeblogCategory], scenario_groups: list[ScenarioGroup] | None = None, weblog_env: dict[str, str | None] | None = None, @@ -567,6 +567,7 @@ def __init__( name: str, *, doc: str, + github_workflow: str | None = "endtoend", additional_trace_header_tags: tuple[str, ...] = (), agent_env: dict[str, str | None] | None = None, appsec_enabled: bool = True, @@ -613,6 +614,7 @@ def __init__( rc_backend_enabled=rc_backend_enabled, require_api_key=require_api_key, flush_weblog_on_stop=flush_weblog_on_stop, + github_workflow=github_workflow, runtime_metrics_enabled=runtime_metrics_enabled, scenario_groups=scenario_groups, span_events=span_events, @@ -632,11 +634,11 @@ class FeatureFlaggingAgentlessEndToEndScenario(DdTraceEndToEndScenario): _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None def configure(self, config: pytest.Config) -> None: - if not self.replay: - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") - self._start_mock_backend(worker_id) - try: + if not self.replay: + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + self._start_mock_backend(worker_id) + super().configure(config) except BaseException: self._stop_mock_backend() @@ -661,12 +663,15 @@ def mock_backend_status(self) -> MockFFEAgentlessBackendStatus | None: return self._last_mock_backend_status def _stop_mock_backend(self) -> None: - if self._mock_backend is None: + backend = self._mock_backend + if backend is None: return - self._last_mock_backend_status = self._mock_backend.status() - self._mock_backend.close() self._mock_backend = None + try: + self._last_mock_backend_status = backend.status() + finally: + backend.close() def close_targets(self) -> None: try: From eb57dc60db0956e740cc2bd2bc40a83762c0311c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 14 Jul 2026 11:08:06 -0400 Subject: [PATCH 7/9] test(ffe): use manifest-only activation --- tests/test_the_test/test_group_rules.py | 1 - utils/_context/_scenarios/__init__.py | 1 - utils/_context/_scenarios/endtoend.py | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index 083033fe03f..d6b3b18ec93 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -18,7 +18,6 @@ def test_tracer_release(): # make an exclusion list dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless - assert dormant_agentless_scenario.github_workflow is None assert dormant_agentless_scenario.include_agent is False assert dormant_agentless_scenario._flush_weblog_on_stop is False # noqa: SLF001 - no-Agent teardown contract assert dormant_agentless_scenario.use_proxy is False diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 3fe0f610ea4..1764bc43436 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -739,7 +739,6 @@ class _Scenarios: include_agent=False, include_default_scenario_groups=False, flush_weblog_on_stop=False, - github_workflow=None, library_interface_timeout=0, scenario_groups=[scenario_groups.ffe], use_proxy_for_agent=False, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index c679a08ce94..736410da6e4 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -200,7 +200,7 @@ def __init__( name: str, *, doc: str, - github_workflow: str | None = "endtoend", + github_workflow: str = "endtoend", weblog_categories: list[WeblogCategory], scenario_groups: list[ScenarioGroup] | None = None, weblog_env: dict[str, str | None] | None = None, @@ -567,7 +567,6 @@ def __init__( name: str, *, doc: str, - github_workflow: str | None = "endtoend", additional_trace_header_tags: tuple[str, ...] = (), agent_env: dict[str, str | None] | None = None, appsec_enabled: bool = True, @@ -614,7 +613,6 @@ def __init__( rc_backend_enabled=rc_backend_enabled, require_api_key=require_api_key, flush_weblog_on_stop=flush_weblog_on_stop, - github_workflow=github_workflow, runtime_metrics_enabled=runtime_metrics_enabled, scenario_groups=scenario_groups, span_events=span_events, From c9f955e56e9fd4e5201dfb18fe1994b611d67b01 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 14 Jul 2026 12:58:53 -0400 Subject: [PATCH 8/9] test(ffe): link agentless manifests to Jira --- manifests/cpp_httpd.yml | 2 +- manifests/cpp_nginx.yml | 2 +- manifests/dotnet.yml | 2 +- manifests/golang.yml | 2 +- manifests/java.yml | 2 +- manifests/nodejs.yml | 2 +- manifests/php.yml | 2 +- manifests/python.yml | 2 +- manifests/ruby.yml | 2 +- manifests/rust.yml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 6cbddd016f0..e427a3e3ce0 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -39,7 +39,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario is not enabled for C++) + tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 74357e1279e..af4d076421e 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -269,7 +269,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario is not enabled for C++) + tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e80fd4b38b9..ec83f5c4531 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -720,7 +720,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <3.22.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) diff --git a/manifests/golang.yml b/manifests/golang.yml index fcd91cd9a1b..00d7acef776 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1119,7 +1119,7 @@ manifest: tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: missing_feature (extended event schema not yet shipped) tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0 - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 diff --git a/manifests/java.yml b/manifests/java.yml index 629f9163c3a..822548340b5 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3264,7 +3264,7 @@ manifest: tests/ffe/test_agentless_configuration.py: - weblog_declaration: "*": irrelevant - spring-boot: missing_feature (agentless end-to-end scenario not enabled yet) + spring-boot: missing_feature (FFL-2693 tracks Java agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 481b89e1f56..3c426a65abb 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1707,7 +1707,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2697 tracks Node.js agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/php.yml b/manifests/php.yml index ddb760953b5..7c5ac9320a1 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -711,7 +711,7 @@ manifest: component_version: <1.12.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index c73689d025c..bed8d7c6acf 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1345,7 +1345,7 @@ manifest: tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2699 tracks Python agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev diff --git a/manifests/ruby.yml b/manifests/ruby.yml index e892bb753ab..b527cf62864 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1931,7 +1931,7 @@ manifest: - declaration: missing_feature (Not implemented) component_version: <2.7.0 tests/docker_ssi/test_docker_ssi_servicenaming.py::TestDockerServiceNaming::test_service_name: missing_feature (No implemented) - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index 17a9c8a6a41..32be24793de 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -23,7 +23,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature - tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage) + tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) From 36a971076bb8312b9dd6e612d1fa17ba7fa58800 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 14:33:31 -0400 Subject: [PATCH 9/9] test(ffe): address agentless harness review feedback --- .github/workflows/run-end-to-end.yml | 3 + tests/ffe/test_agentless_configuration.py | 12 +-- .../test_ffe/test_configuration_sources.py | 4 +- .../test_compute_libraries_and_scenarios.py | 24 ++++++ tests/test_the_test/test_group_rules.py | 1 - .../test_mock_ffe_agentless_backend.py | 73 ++----------------- utils/_context/_scenarios/__init__.py | 16 +--- utils/_context/_scenarios/endtoend.py | 65 +++++++++++------ utils/_context/weblog_infrastructure.py | 7 +- utils/mocked_backend/__init__.py | 1 + .../ffe.py} | 2 +- .../scripts/libraries_and_scenarios_rules.yml | 4 + 12 files changed, 94 insertions(+), 118 deletions(-) create mode 100644 utils/mocked_backend/__init__.py rename utils/{docker_fixtures/_mock_ffe_agentless_backend.py => mocked_backend/ffe.py} (99%) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 9415bc5a3cd..a830e1d1c4b 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -320,6 +320,9 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION + - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') + run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py index 2e5ef64447b..38fd8005221 100644 --- a/tests/ffe/test_agentless_configuration.py +++ b/tests/ffe/test_agentless_configuration.py @@ -3,7 +3,7 @@ import json from utils import features, scenarios, weblog -from utils.docker_fixtures._mock_ffe_agentless_backend import CONFIG_PATH +from utils.mocked_backend.ffe import CONFIG_PATH @scenarios.feature_flagging_and_experimentation_agentless @@ -20,13 +20,13 @@ def setup_default_agentless_source(self) -> None: "attributes": {}, }, ) - self.backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() def test_default_agentless_source(self) -> None: assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" assert json.loads(self.response.text)["value"] == "on-value" - assert self.backend_status is not None - assert self.backend_status["requests_total"] >= 1 - assert self.backend_status["last_path"] == CONFIG_PATH - assert self.backend_status["last_auth_present"] is True + backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() + assert backend_status is not None + assert backend_status["requests_total"] >= 1 + assert backend_status["last_path"] == CONFIG_PATH + assert backend_status["last_auth_present"] is True diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index f6cf26431ac..eea10e524ec 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -24,14 +24,14 @@ from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI -from utils.docker_fixtures._mock_ffe_agentless_backend import ( +from utils.mocked_backend.ffe import ( CONFIG_PATH, MockFFEAgentlessBackendServer, MockFFEAgentlessBackendStatus, ) parametrize = pytest.mark.parametrize -pytest_plugins = ["utils.docker_fixtures._mock_ffe_agentless_backend"] +pytest_plugins = ["utils.mocked_backend.ffe"] UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" RC_PRODUCT = "FFE_FLAGS" diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 72108957cf4..04efbb76fcb 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -468,6 +468,30 @@ def test_agentless_ffe_test_file(self): "", ) + def test_agentless_ffe_mocked_backend_file(self): + inputs = build_inputs(modified_files=["utils/mocked_backend/ffe.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS,PARAMETRIC", + "", + ) + + def test_end_to_end_scenario_framework_file(self): + inputs = build_inputs(modified_files=["utils/_context/_scenarios/endtoend.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "end_to_end", + ) + def test_json_modification(self): inputs = build_inputs(modified_files=["tests/debugger/utils/probe_snapshot_log_line.json"]) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index d6b3b18ec93..dda1b74ffbf 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -19,7 +19,6 @@ def test_tracer_release(): dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless assert dormant_agentless_scenario.include_agent is False - assert dormant_agentless_scenario._flush_weblog_on_stop is False # noqa: SLF001 - no-Agent teardown contract assert dormant_agentless_scenario.use_proxy is False assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index 0511085f5a0..4327388ea5e 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -5,10 +5,10 @@ import requests import pytest -from utils import features, interfaces, scenarios +from utils import scenarios from utils._context._scenarios import endtoend as endtoend_scenarios from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment -from utils.docker_fixtures._mock_ffe_agentless_backend import ( +from utils.mocked_backend.ffe import ( CONFIG_PATH, EXPECTED_API_KEY, MockFFEAgentlessBackendServer, @@ -17,7 +17,6 @@ @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id: str) -> None: server = MockFFEAgentlessBackendServer(worker_id) try: @@ -34,7 +33,6 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.MonkeyPatch, worker_id: str) -> None: monkeypatch.delenv("SYSTEM_TESTS_MOCK_FFE_AGENTLESS_BACKEND_BASE_URL", raising=False) monkeypatch.delenv("SYSTEM_TESTS_MOCK_AGENTLESS_BACKEND_BASE_URL", raising=False) @@ -50,7 +48,6 @@ def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.Mon @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> None: server = MockFFEAgentlessBackendServer(worker_id) try: @@ -73,19 +70,12 @@ def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> N @scenarios.test_the_test -@features.not_reported -def test_agentless_end_to_end_scenario_starts_backend_before_weblog(worker_id: str) -> None: - scenario = FeatureFlaggingAgentlessEndToEndScenario( - "MOCK_FFE_AGENTLESS_E2E", - doc="test", - include_agent=False, - use_proxy_for_agent=False, - use_proxy_for_weblog=False, - ) +def test_agentless_end_to_end_scenario_starts_backend_before_weblog() -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_E2E", doc="test") try: assert scenario.agent_container not in scenario._containers # noqa: SLF001 - focused topology test - scenario._start_mock_backend(worker_id) # noqa: SLF001 - focused lifecycle test + scenario._start_mock_backend() # noqa: SLF001 - focused lifecycle test environment = scenario.weblog_infra.library_container.environment assert "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE" not in environment @@ -103,54 +93,14 @@ def test_agentless_end_to_end_scenario_starts_backend_before_weblog(worker_id: s @scenarios.test_the_test -@features.not_reported -def test_agentless_end_to_end_scenario_stops_without_tracer_flush(monkeypatch: pytest.MonkeyPatch) -> None: - scenario = FeatureFlaggingAgentlessEndToEndScenario( - "MOCK_FFE_AGENTLESS_NO_FLUSH", - doc="test", - flush_weblog_on_stop=False, - include_agent=False, - use_proxy_for_agent=False, - use_proxy_for_weblog=False, - ) - flushed = False - stopped = False - - def flush() -> None: - nonlocal flushed - flushed = True - - def stop() -> None: - nonlocal stopped - stopped = True - - monkeypatch.setattr(scenario, "_wait_interface", lambda *_: None) - monkeypatch.setattr(scenario.weblog_infra.http_container, "flush", flush) - monkeypatch.setattr(scenario.weblog_infra.http_container, "stop", stop) - monkeypatch.setattr(interfaces.library, "check_deserialization_errors", lambda: None) - - scenario._wait_and_stop_containers(force_interface_timout_to_zero=True) # noqa: SLF001 - - assert stopped is True - assert flushed is False - - -@scenarios.test_the_test -@features.not_reported def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: - scenario = FeatureFlaggingAgentlessEndToEndScenario( - "MOCK_FFE_AGENTLESS_STARTUP_FAILURE", - doc="test", - include_agent=False, - use_proxy_for_agent=False, - use_proxy_for_weblog=False, - ) + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STARTUP_FAILURE", doc="test") backend = MagicMock(spec=MockFFEAgentlessBackendServer) backend.reset.side_effect = RuntimeError("reset failed") - def create_backend(_worker_id: str) -> MagicMock: + def create_backend() -> MagicMock: return backend monkeypatch.setattr(endtoend_scenarios, "MockFFEAgentlessBackendServer", create_backend) @@ -163,15 +113,8 @@ def create_backend(_worker_id: str) -> MagicMock: @scenarios.test_the_test -@features.not_reported def test_agentless_end_to_end_scenario_closes_backend_when_status_fails() -> None: - scenario = FeatureFlaggingAgentlessEndToEndScenario( - "MOCK_FFE_AGENTLESS_STATUS_FAILURE", - doc="test", - include_agent=False, - use_proxy_for_agent=False, - use_proxy_for_weblog=False, - ) + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STATUS_FAILURE", doc="test") backend = MagicMock(spec=MockFFEAgentlessBackendServer) backend.status.side_effect = RuntimeError("status failed") scenario._mock_backend = backend # noqa: SLF001 - focused lifecycle test diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 1764bc43436..b24fcc8a1de 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -728,21 +728,7 @@ class _Scenarios: ) feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario( - "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", - weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", - "DD_REMOTE_CONFIGURATION_ENABLED": "false", - }, - doc="Validate default agentless UFC delivery and evaluation without a Datadog Agent.", - include_agent=False, - include_default_scenario_groups=False, - flush_weblog_on_stop=False, - library_interface_timeout=0, - scenario_groups=[scenario_groups.ffe], - use_proxy_for_agent=False, - use_proxy_for_weblog=False, + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" ) remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 736410da6e4..af448f595c3 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -23,7 +23,7 @@ ) from utils._context.weblog_infrastructure import EndToEndWeblogInfra from utils.docker_fixtures._core import extra_hosts_for_environment -from utils.docker_fixtures._mock_ffe_agentless_backend import ( +from utils.mocked_backend.ffe import ( EXPECTED_API_KEY, MockFFEAgentlessBackendServer, MockFFEAgentlessBackendStatus, @@ -195,6 +195,12 @@ def close_targets(self): class EndToEndScenario(DockerScenario): """Scenario with an instrumented HTTP application and an optional Datadog Agent.""" + _default_scenario_groups: tuple[ScenarioGroup, ...] = ( + all_scenario_groups.all, + all_scenario_groups.end_to_end, + all_scenario_groups.tracer_release, + ) + def __init__( self, name: str, @@ -225,22 +231,11 @@ def __init__( backend_interface_timeout: int = 0, include_buddies: bool = False, include_agent: bool = True, - include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, require_api_key: bool = False, - flush_weblog_on_stop: bool = True, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: - default_scenario_groups = ( - [ - all_scenario_groups.all, - all_scenario_groups.end_to_end, - all_scenario_groups.tracer_release, - ] - if include_default_scenario_groups - else [] - ) - scenario_groups = default_scenario_groups + (scenario_groups or []) + scenario_groups = [*self._default_scenario_groups, *(scenario_groups or [])] super().__init__( name, @@ -264,7 +259,6 @@ def __init__( self.include_agent = include_agent self._use_proxy_for_agent = include_agent and use_proxy_for_agent self._use_proxy_for_weblog = use_proxy_for_weblog - self._flush_weblog_on_stop = flush_weblog_on_stop self._require_api_key = require_api_key self.agent_container = AgentContainer( @@ -398,7 +392,7 @@ def _start_interfaces_watchdog(self): open_telemetry_interfaces: list[ProxyBasedInterfaceValidator] = ( [interfaces.open_telemetry] if self.include_opentelemetry else [] ) - agent_interfaces = [interfaces.agent] if self.include_agent else [] + agent_interfaces: list[ProxyBasedInterfaceValidator] = [interfaces.agent] if self.include_agent else [] super().start_interfaces_watchdog( [interfaces.library] + agent_interfaces @@ -474,7 +468,7 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.library, 0 if force_interface_timout_to_zero else self.library_interface_timeout ) - self.weblog_infra.stop(flush=self._flush_weblog_on_stop) + self.weblog_infra.stop() interfaces.library.check_deserialization_errors() for container in self.buddies: @@ -574,7 +568,6 @@ def __init__( client_drop_p0s: bool | None = None, iast_enabled: bool = True, include_agent: bool = True, - include_default_scenario_groups: bool = True, include_opentelemetry: bool = False, library_interface_timeout: int | None = None, meta_structs_disabled: bool = False, @@ -583,7 +576,6 @@ def __init__( rc_api_enabled: bool = False, rc_backend_enabled: bool = False, require_api_key: bool = False, - flush_weblog_on_stop: bool = True, runtime_metrics_enabled: bool = False, scenario_groups: list[ScenarioGroup] | None = None, span_events: bool = True, @@ -603,7 +595,6 @@ def __init__( doc=doc, iast_enabled=iast_enabled, include_agent=include_agent, - include_default_scenario_groups=include_default_scenario_groups, include_opentelemetry=include_opentelemetry, library_interface_timeout=library_interface_timeout, meta_structs_disabled=meta_structs_disabled, @@ -612,7 +603,6 @@ def __init__( rc_api_enabled=rc_api_enabled, rc_backend_enabled=rc_backend_enabled, require_api_key=require_api_key, - flush_weblog_on_stop=flush_weblog_on_stop, runtime_metrics_enabled=runtime_metrics_enabled, scenario_groups=scenario_groups, span_events=span_events, @@ -628,24 +618,51 @@ def __init__( class FeatureFlaggingAgentlessEndToEndScenario(DdTraceEndToEndScenario): """FFE end-to-end scenario with UFC available before the weblog starts.""" + _default_scenario_groups: tuple[ScenarioGroup, ...] = () + _mock_backend: MockFFEAgentlessBackendServer | None = None _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None + def __init__( + self, + name: str, + *, + doc: str = "Validate default agentless UFC delivery and evaluation without a Datadog Agent.", + weblog_env: dict[str, str | None] | None = None, + ) -> None: + environment: dict[str, str | None] = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + } + environment.update(weblog_env or {}) + + super().__init__( + name, + doc=doc, + include_agent=False, + library_interface_timeout=0, + scenario_groups=[all_scenario_groups.ffe], + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + weblog_env=environment, + ) + def configure(self, config: pytest.Config) -> None: try: if not self.replay: - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") - self._start_mock_backend(worker_id) + self._start_mock_backend() super().configure(config) except BaseException: self._stop_mock_backend() raise - def _start_mock_backend(self, worker_id: str) -> None: + def _start_mock_backend(self) -> None: assert self._mock_backend is None, "mock FFE agentless backend is already running" - self._mock_backend = MockFFEAgentlessBackendServer(worker_id) + self._mock_backend = MockFFEAgentlessBackendServer() self._mock_backend.reset() environment = self.weblog_infra.library_container.environment diff --git a/utils/_context/weblog_infrastructure.py b/utils/_context/weblog_infrastructure.py index 175ac6a95f5..3796727dff3 100644 --- a/utils/_context/weblog_infrastructure.py +++ b/utils/_context/weblog_infrastructure.py @@ -33,7 +33,7 @@ def configure(self, config: pytest.Config) -> None: """Perform any configuration. Executed only if the weblog will be used""" @abstractmethod - def stop(self, *, flush: bool = True) -> None: + def stop(self) -> None: """Stop the tested infra""" @@ -208,7 +208,7 @@ def get_containers(self) -> tuple[TestedContainer, ...]: ) return (self.http_container, *self._other_containers) - def stop(self, *, flush: bool = True) -> None: + def stop(self) -> None: if self._is_proxy_weblog: if self._proxy_runtime_container: self._proxy_runtime_container.stop() @@ -217,8 +217,7 @@ def stop(self, *, flush: bool = True) -> None: if self._dummy_server_container: self._dummy_server_container.stop() else: - if flush: - self.http_container.flush() + self.http_container.flush() self.http_container.stop() @property diff --git a/utils/mocked_backend/__init__.py b/utils/mocked_backend/__init__.py new file mode 100644 index 00000000000..272b27ab808 --- /dev/null +++ b/utils/mocked_backend/__init__.py @@ -0,0 +1 @@ +"""Reusable mocked backend implementations for system-test scenarios.""" diff --git a/utils/docker_fixtures/_mock_ffe_agentless_backend.py b/utils/mocked_backend/ffe.py similarity index 99% rename from utils/docker_fixtures/_mock_ffe_agentless_backend.py rename to utils/mocked_backend/ffe.py index d7076eef17e..d1cf5fae087 100644 --- a/utils/docker_fixtures/_mock_ffe_agentless_backend.py +++ b/utils/mocked_backend/ffe.py @@ -273,7 +273,7 @@ def _strip_config_path(url: str) -> str: class MockFFEAgentlessBackendServer: - def __init__(self, worker_id: str) -> None: + def __init__(self, worker_id: str = "master") -> None: self.port = get_host_port(worker_id, 4900) self._server = MockFFEAgentlessBackendHTTPServer(("0.0.0.0", self.port)) # noqa: S104 - test fixture must be container-reachable. self._thread = threading.Thread( diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index 11ed0361615..1423d9a9cb1 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -224,6 +224,7 @@ patterns: scenario_groups: appsec_rasp_scenario utils/_context/_scenarios/endtoend.py: scenario_groups: end_to_end + scenarios: FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS utils/_context/_scenarios/integration_frameworks.py: scenarios: INTEGRATION_FRAMEWORKS utils/_context/_scenarios/integrations.py: @@ -289,6 +290,9 @@ patterns: utils/interfaces/_library/*: scenario_groups: [end_to_end, open_telemetry] + utils/mocked_backend/ffe.py: + scenarios: [FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS, PARAMETRIC] + utils/_context/virtual_machine.py: scenario_groups: onboarding