diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 9415bc5a3cd..a830e1d1c4b 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -320,6 +320,9 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION + - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') + run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 271b8f6aacc..e427a3e3ce0 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -39,6 +39,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index d40ad5cb7f2..2d496643cf5 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -270,6 +270,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature + tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e96360ad4fd..aa0205965c5 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -721,6 +721,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <3.22.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2703 tracks .NET agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) diff --git a/manifests/golang.yml b/manifests/golang.yml index 29ddc54a0f2..fc264795803 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1120,6 +1120,7 @@ manifest: tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: missing_feature (extended event schema not yet shipped) tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0 + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 diff --git a/manifests/java.yml b/manifests/java.yml index 4d1cd0cc716..5e5f23b011f 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3262,6 +3262,10 @@ manifest: component_version: <1.52.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: 1.61.0-SNAPSHOT # Normalization of telemetry keys updated in https://github.com/DataDog/dd-trace-java/pull/10823 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) + tests/ffe/test_agentless_configuration.py: + - weblog_declaration: + "*": irrelevant + spring-boot: missing_feature (FFL-2693 tracks Java agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 8051d6a7457..ca6d18c4a71 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1708,6 +1708,7 @@ manifest: - declaration: missing_feature (Not implemented yet) component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2697 tracks Node.js agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/php.yml b/manifests/php.yml index 2b00db41ccf..11f72432e70 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -740,6 +740,7 @@ manifest: component_version: <1.12.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index 3445067e84d..f69ddb0b009 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1346,6 +1346,7 @@ manifest: tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2699 tracks Python agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 45052c8c11c..915773d2d1c 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1940,6 +1940,7 @@ manifest: - declaration: missing_feature (Not implemented) component_version: <2.7.0 tests/docker_ssi/test_docker_ssi_servicenaming.py::TestDockerServiceNaming::test_service_name: missing_feature (No implemented) + tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index 02a875c1482..32be24793de 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -23,6 +23,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature + tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/tests/ffe/test_agentless_configuration.py b/tests/ffe/test_agentless_configuration.py new file mode 100644 index 00000000000..38fd8005221 --- /dev/null +++ b/tests/ffe/test_agentless_configuration.py @@ -0,0 +1,32 @@ +"""Test default agentless UFC delivery before FFE evaluation.""" + +import json + +from utils import features, scenarios, weblog +from utils.mocked_backend.ffe import CONFIG_PATH + + +@scenarios.feature_flagging_and_experimentation_agentless +@features.feature_flags_agentless +class Test_FFE_Agentless_Configuration: + def setup_default_agentless_source(self) -> None: + self.response = weblog.post( + "/ffe", + json={ + "flag": "empty-targeting-key-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + + def test_default_agentless_source(self) -> None: + assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" + assert json.loads(self.response.text)["value"] == "on-value" + + backend_status = scenarios.feature_flagging_and_experimentation_agentless.mock_backend_status() + assert backend_status is not None + assert backend_status["requests_total"] >= 1 + assert backend_status["last_path"] == CONFIG_PATH + assert backend_status["last_auth_present"] is True diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index f6cf26431ac..eea10e524ec 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -24,14 +24,14 @@ from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI -from utils.docker_fixtures._mock_ffe_agentless_backend import ( +from utils.mocked_backend.ffe import ( CONFIG_PATH, MockFFEAgentlessBackendServer, MockFFEAgentlessBackendStatus, ) parametrize = pytest.mark.parametrize -pytest_plugins = ["utils.docker_fixtures._mock_ffe_agentless_backend"] +pytest_plugins = ["utils.mocked_backend.ffe"] UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" RC_PRODUCT = "FFE_FLAGS" diff --git a/tests/test_the_test/scenarios.json b/tests/test_the_test/scenarios.json index 5967b859045..43a32cc149f 100644 --- a/tests/test_the_test/scenarios.json +++ b/tests/test_the_test/scenarios.json @@ -3348,6 +3348,9 @@ "tests/external_processing/test_apm.py::Test_ExternalProcessing_Tracing::test_correct_span_structure": [ "EXTERNAL_PROCESSING" ], + "tests/ffe/test_agentless_configuration.py::Test_FFE_Agentless_Configuration::test_default_agentless_source": [ + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" + ], "tests/integrations/test_cassandra.py::Test_Cassandra::test_main": [ "INTEGRATIONS" ], @@ -5917,4 +5920,4 @@ "tests/test_the_test/test_version.py::test_php_version": [ "TEST_THE_TEST" ] -} \ No newline at end of file +} diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index aaf6b694798..04efbb76fcb 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -456,6 +456,42 @@ def test_otel_test_file(self): "", ) + def test_agentless_ffe_test_file(self): + inputs = build_inputs(modified_files=["tests/ffe/test_agentless_configuration.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "", + ) + + def test_agentless_ffe_mocked_backend_file(self): + inputs = build_inputs(modified_files=["utils/mocked_backend/ffe.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS,PARAMETRIC", + "", + ) + + def test_end_to_end_scenario_framework_file(self): + inputs = build_inputs(modified_files=["utils/_context/_scenarios/endtoend.py"]) + assert_github_processor( + inputs, + default_libs_with_prod, + default_libs_with_dev, + 3600, + "false", + "DEFAULT,FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "end_to_end", + ) + def test_json_modification(self): inputs = build_inputs(modified_files=["tests/debugger/utils/probe_snapshot_log_line.json"]) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index b636ea46a4a..dda1b74ffbf 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -17,9 +17,18 @@ def test_appsec(): def test_tracer_release(): # make an exclusion list + dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless + assert dormant_agentless_scenario.include_agent is False + assert dormant_agentless_scenario.use_proxy is False + assert scenario_groups.ffe in dormant_agentless_scenario.scenario_groups + assert scenario_groups.all not in dormant_agentless_scenario.scenario_groups + assert scenario_groups.end_to_end not in dormant_agentless_scenario.scenario_groups + assert scenario_groups.tracer_release not in dormant_agentless_scenario.scenario_groups + not_in_tracer_release_group = [ # list of scenario that will never be part of tracer release scenarios.fuzzer, + dormant_agentless_scenario, scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, diff --git a/tests/test_the_test/test_mock_ffe_agentless_backend.py b/tests/test_the_test/test_mock_ffe_agentless_backend.py index e6f5f4d93d1..6da65a9530f 100644 --- a/tests/test_the_test/test_mock_ffe_agentless_backend.py +++ b/tests/test_the_test/test_mock_ffe_agentless_backend.py @@ -1,11 +1,14 @@ """Unit coverage for the mock FFE agentless backend test fixture.""" +from unittest.mock import MagicMock + import requests import pytest -from utils import features, scenarios +from utils import scenarios +from utils._context._scenarios import endtoend as endtoend_scenarios from utils.docker_fixtures._core import HOST_GATEWAY_EXTRA_HOSTS, extra_hosts_for_environment -from utils.docker_fixtures._mock_ffe_agentless_backend import ( +from utils.mocked_backend.ffe import ( CONFIG_PATH, CONFIG_QUERY, EXPECTED_API_KEY, @@ -13,10 +16,10 @@ MockFFEAgentlessBackendServer, UFC_RESPONSE_TYPE, ) +from utils._context._scenarios.endtoend import FeatureFlaggingAgentlessEndToEndScenario @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id: str) -> None: server = MockFFEAgentlessBackendServer(worker_id, port=0) try: @@ -50,7 +53,6 @@ def test_mock_ffe_agentless_backend_serves_fixture_and_tracks_metadata(worker_id @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.MonkeyPatch, worker_id: str) -> None: monkeypatch.delenv("SYSTEM_TESTS_MOCK_FFE_AGENTLESS_BACKEND_BASE_URL", raising=False) monkeypatch.delenv("SYSTEM_TESTS_MOCK_AGENTLESS_BACKEND_BASE_URL", raising=False) @@ -67,7 +69,6 @@ def test_mock_ffe_agentless_backend_host_gateway_mapping(monkeypatch: pytest.Mon @scenarios.test_the_test -@features.not_reported def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> None: server = MockFFEAgentlessBackendServer(worker_id, port=0) try: @@ -87,3 +88,60 @@ def test_mock_ffe_agentless_backend_status_is_metadata_only(worker_id: str) -> N assert "body" not in status finally: server.close() + + +@scenarios.test_the_test +def test_agentless_end_to_end_scenario_starts_backend_before_weblog() -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_E2E", doc="test") + + try: + assert scenario.agent_container not in scenario._containers # noqa: SLF001 - focused topology test + scenario._start_mock_backend() # noqa: SLF001 - focused lifecycle test + + environment = scenario.weblog_infra.library_container.environment + assert "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE" not in environment + assert "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" not in environment + base_url = environment["DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"] + assert isinstance(base_url, str) + assert base_url.endswith(f"{CONFIG_PATH}?{CONFIG_QUERY}") + assert scenario.weblog_infra.library_container.extra_hosts == HOST_GATEWAY_EXTRA_HOSTS + + status = scenario.mock_backend_status() + assert status is not None + assert status["requests_total"] == 0 + finally: + scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test + + +@scenarios.test_the_test +def test_agentless_end_to_end_scenario_closes_backend_when_startup_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STARTUP_FAILURE", doc="test") + backend = MagicMock(spec=MockFFEAgentlessBackendServer) + backend.reset.side_effect = RuntimeError("reset failed") + + def create_backend() -> MagicMock: + return backend + + monkeypatch.setattr(endtoend_scenarios, "MockFFEAgentlessBackendServer", create_backend) + + with pytest.raises(RuntimeError, match="reset failed"): + scenario.configure(MagicMock(spec=pytest.Config)) + + backend.close.assert_called_once_with() + assert scenario._mock_backend is None # noqa: SLF001 - focused lifecycle test + + +@scenarios.test_the_test +def test_agentless_end_to_end_scenario_closes_backend_when_status_fails() -> None: + scenario = FeatureFlaggingAgentlessEndToEndScenario("MOCK_FFE_AGENTLESS_STATUS_FAILURE", doc="test") + backend = MagicMock(spec=MockFFEAgentlessBackendServer) + backend.status.side_effect = RuntimeError("status failed") + scenario._mock_backend = backend # noqa: SLF001 - focused lifecycle test + + with pytest.raises(RuntimeError, match="status failed"): + scenario._stop_mock_backend() # noqa: SLF001 - focused lifecycle test + + backend.close.assert_called_once_with() + assert scenario._mock_backend is None # noqa: SLF001 - focused lifecycle test diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index e7f9b94f88e..0c1fd93232c 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -7,7 +7,12 @@ from .aws_lambda import LambdaScenario from .core import Scenario, scenario_groups from .default import DefaultScenario -from .endtoend import DockerScenario, DdTraceEndToEndScenario, GraphQlEndToEndScenario +from .endtoend import ( + DockerScenario, + DdTraceEndToEndScenario, + FeatureFlaggingAgentlessEndToEndScenario, + GraphQlEndToEndScenario, +) from .integrations import ( CrossedTracingLibraryScenario, DbmDynamicServiceScenario, @@ -727,6 +732,10 @@ class _Scenarios: scenario_groups=[scenario_groups.ffe], ) + feature_flagging_and_experimentation_agentless = FeatureFlaggingAgentlessEndToEndScenario( + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS" + ) + remote_config_mocked_backend_asm_features_nocache = DdTraceEndToEndScenario( "REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE", rc_api_enabled=True, diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index bda32d3c44f..af448f595c3 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -12,6 +12,7 @@ from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.buddies import BuddyHostPorts from utils.proxy.ports import ProxyPorts +from utils._context.component_version import Version from utils._context.docker import get_docker_client from utils._context.containers import ( WeblogContainer, @@ -21,6 +22,12 @@ TestedContainer, ) from utils._context.weblog_infrastructure import EndToEndWeblogInfra +from utils.docker_fixtures._core import extra_hosts_for_environment +from utils.mocked_backend.ffe import ( + EXPECTED_API_KEY, + MockFFEAgentlessBackendServer, + MockFFEAgentlessBackendStatus, +) from utils._context.constants import WeblogCategory from utils._logger import logger @@ -186,7 +193,13 @@ def close_targets(self): class EndToEndScenario(DockerScenario): - """Scenario that implier an instrumented HTTP application shipping a datadog tracer (weblog) and an datadog agent""" + """Scenario with an instrumented HTTP application and an optional Datadog Agent.""" + + _default_scenario_groups: tuple[ScenarioGroup, ...] = ( + all_scenario_groups.all, + all_scenario_groups.end_to_end, + all_scenario_groups.tracer_release, + ) def __init__( self, @@ -217,15 +230,12 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_agent: bool = True, include_opentelemetry: bool = False, require_api_key: bool = False, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: - scenario_groups = [ - all_scenario_groups.all, - all_scenario_groups.end_to_end, - all_scenario_groups.tracer_release, - ] + (scenario_groups or []) + scenario_groups = [*self._default_scenario_groups, *(scenario_groups or [])] super().__init__( name, @@ -234,7 +244,7 @@ def __init__( scenario_groups=scenario_groups, weblog_categories=weblog_categories, enable_ipv6=enable_ipv6, - use_proxy=use_proxy_for_agent or use_proxy_for_weblog, + use_proxy=(include_agent and use_proxy_for_agent) or use_proxy_for_weblog, rc_api_enabled=rc_api_enabled, rc_backend_enabled=rc_backend_enabled, meta_structs_disabled=meta_structs_disabled, @@ -243,14 +253,19 @@ def __init__( obfuscation_version=obfuscation_version, ) - self._use_proxy_for_agent = use_proxy_for_agent + if include_buddies and not include_agent: + raise ValueError("include_buddies requires include_agent") + + self.include_agent = include_agent + self._use_proxy_for_agent = include_agent and use_proxy_for_agent self._use_proxy_for_weblog = use_proxy_for_weblog self._require_api_key = require_api_key self.agent_container = AgentContainer( use_proxy=use_proxy_for_agent, rc_backend_enabled=rc_backend_enabled, environment=agent_env ) - self._containers.append(self.agent_container) + if include_agent: + self._containers.append(self.agent_container) self._weblog_env = dict(weblog_env) if weblog_env else {} self.weblog_infra = EndToEndWeblogInfra( @@ -315,12 +330,14 @@ def configure(self, config: pytest.Config): self._set_containers_dependancies() super().configure(config) - interfaces.agent.configure(self.host_log_folder, replay=self.replay) + if self.include_agent: + interfaces.agent.configure(self.host_log_folder, replay=self.replay) interfaces.library.configure(self.host_log_folder, replay=self.replay) interfaces.backend.configure(self.host_log_folder, replay=self.replay) interfaces.library_dotnet_managed.configure(self.host_log_folder, replay=self.replay) interfaces.library_stdout.configure(self.host_log_folder, replay=self.replay) - interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) + if self.include_agent: + interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) if self.include_opentelemetry: interfaces.open_telemetry.configure(self.host_log_folder, replay=self.replay) @@ -365,7 +382,8 @@ def _set_containers_dependancies(self) -> None: self.agent_container.depends_on.append(self.proxy_container) proxy_container = self.proxy_container if self._use_proxy_for_weblog else None - self.weblog_infra.set_weblog_dependencies(self.agent_container, proxy_container) + agent_container = self.agent_container if self.include_agent else None + self.weblog_infra.set_weblog_dependencies(agent_container, proxy_container) for buddy in self.buddies: buddy.depends_on.append(self.agent_container) @@ -374,8 +392,10 @@ def _start_interfaces_watchdog(self): open_telemetry_interfaces: list[ProxyBasedInterfaceValidator] = ( [interfaces.open_telemetry] if self.include_opentelemetry else [] ) + agent_interfaces: list[ProxyBasedInterfaceValidator] = [interfaces.agent] if self.include_agent else [] super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [interfaces.library] + + agent_interfaces + [container.interface for container in self.buddies] + open_telemetry_interfaces ) @@ -385,7 +405,8 @@ def _set_weblog_domain(self): self.weblog_container.set_weblog_domain_for_ipv6(self._network) def _set_components(self): - self.components["agent"] = self.agent_version + if self.include_agent: + self.components["agent"] = self.agent_version self.components["library"] = self.library.version self.components[self.library.name] = self.library.version @@ -432,8 +453,9 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): container.interface.load_data_from_logs() container.interface.check_deserialization_errors() - interfaces.agent.load_data_from_logs() - interfaces.agent.check_deserialization_errors() + if self.include_agent: + interfaces.agent.load_data_from_logs() + interfaces.agent.check_deserialization_errors() interfaces.backend.load_data_from_logs() @@ -455,11 +477,12 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): container.stop() container.interface.check_deserialization_errors() - self._wait_interface( - interfaces.agent, 0 if force_interface_timout_to_zero else self.agent_interface_timeout - ) - self.agent_container.stop() - interfaces.agent.check_deserialization_errors() + if self.include_agent: + self._wait_interface( + interfaces.agent, 0 if force_interface_timout_to_zero else self.agent_interface_timeout + ) + self.agent_container.stop() + interfaces.agent.check_deserialization_errors() self._wait_interface( interfaces.backend, 0 if force_interface_timout_to_zero else self.backend_interface_timeout @@ -482,7 +505,9 @@ def weblog_container(self) -> WeblogContainer: @property def dd_site(self): - return self.agent_container.dd_site + if self.include_agent: + return self.agent_container.dd_site + return self.weblog_container.environment.get("DD_SITE", "datad0g.com") @property def library(self): @@ -490,7 +515,7 @@ def library(self): @property def agent_version(self): - return self.agent_container.agent_version + return self.agent_container.agent_version if self.include_agent else Version("0.0.0") @property def weblog_variant(self) -> str: @@ -542,6 +567,7 @@ def __init__( backend_interface_timeout: int = 0, client_drop_p0s: bool | None = None, iast_enabled: bool = True, + include_agent: bool = True, include_opentelemetry: bool = False, library_interface_timeout: int | None = None, meta_structs_disabled: bool = False, @@ -554,6 +580,8 @@ def __init__( scenario_groups: list[ScenarioGroup] | None = None, span_events: bool = True, tracer_sampling_rate: float | None = None, + use_proxy_for_agent: bool = True, + use_proxy_for_weblog: bool = True, weblog_env: dict[str, str | None] | None = None, weblog_volumes: dict | None = None, ) -> None: @@ -566,6 +594,7 @@ def __init__( client_drop_p0s=client_drop_p0s, doc=doc, iast_enabled=iast_enabled, + include_agent=include_agent, include_opentelemetry=include_opentelemetry, library_interface_timeout=library_interface_timeout, meta_structs_disabled=meta_structs_disabled, @@ -578,12 +607,94 @@ def __init__( scenario_groups=scenario_groups, span_events=span_events, tracer_sampling_rate=tracer_sampling_rate, + use_proxy_for_agent=use_proxy_for_agent, + use_proxy_for_weblog=use_proxy_for_weblog, weblog_categories=[WeblogCategory.dd_trace], weblog_env=weblog_env, weblog_volumes=weblog_volumes, ) +class FeatureFlaggingAgentlessEndToEndScenario(DdTraceEndToEndScenario): + """FFE end-to-end scenario with UFC available before the weblog starts.""" + + _default_scenario_groups: tuple[ScenarioGroup, ...] = () + + _mock_backend: MockFFEAgentlessBackendServer | None = None + _last_mock_backend_status: MockFFEAgentlessBackendStatus | None = None + + def __init__( + self, + name: str, + *, + doc: str = "Validate default agentless UFC delivery and evaluation without a Datadog Agent.", + weblog_env: dict[str, str | None] | None = None, + ) -> None: + environment: dict[str, str | None] = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": "0.2", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": "2", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + } + environment.update(weblog_env or {}) + + super().__init__( + name, + doc=doc, + include_agent=False, + library_interface_timeout=0, + scenario_groups=[all_scenario_groups.ffe], + use_proxy_for_agent=False, + use_proxy_for_weblog=False, + weblog_env=environment, + ) + + def configure(self, config: pytest.Config) -> None: + try: + if not self.replay: + self._start_mock_backend() + + super().configure(config) + except BaseException: + self._stop_mock_backend() + raise + + def _start_mock_backend(self) -> None: + assert self._mock_backend is None, "mock FFE agentless backend is already running" + + self._mock_backend = MockFFEAgentlessBackendServer() + self._mock_backend.reset() + + environment = self.weblog_infra.library_container.environment + environment |= { + "DD_API_KEY": EXPECTED_API_KEY, + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": self._mock_backend.library_config_url, + } + self.weblog_infra.library_container.extra_hosts = extra_hosts_for_environment(environment) + + def mock_backend_status(self) -> MockFFEAgentlessBackendStatus | None: + if self._mock_backend is not None: + return self._mock_backend.status() + return self._last_mock_backend_status + + def _stop_mock_backend(self) -> None: + backend = self._mock_backend + if backend is None: + return + + self._mock_backend = None + try: + self._last_mock_backend_status = backend.status() + finally: + backend.close() + + def close_targets(self) -> None: + try: + super().close_targets() + finally: + self._stop_mock_backend() + + class GraphQlEndToEndScenario(EndToEndScenario): """Scenario targeting GraphQl feature of a dd-trace library""" diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 59317347c56..8376d418eda 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -22,6 +22,7 @@ from utils._context.docker import get_docker_client from utils._context._image_mirror import mirror_image from utils._context.constants import ContainerPorts +from utils.docker_fixtures._core import extra_hosts_for_environment from utils.proxy.tuf import get_tuf_root_json from utils.proxy.ports import ProxyPorts from utils.proxy.mocked_response import ( @@ -86,6 +87,7 @@ def __init__( cap_add: list[str] | None = None, command: str | list[str] | None = None, environment: dict[str, str | None] | None = None, + extra_hosts: dict[str, str] | None = None, healthcheck: dict | None = None, local_image_only: bool = False, ports: dict | None = None, @@ -115,6 +117,7 @@ def __init__( self.healthy: bool | None = None self.environment = environment or {} + self.extra_hosts = extra_hosts self.volumes = volumes or {} self.ports = ports or {} self.depends_on: list[TestedContainer] = [] @@ -229,6 +232,7 @@ def start(self, network: Network) -> Container: name=self.container_name, hostname=self.name, environment=self.environment, + extra_hosts=self.extra_hosts, # auto_remove=True, detach=True, network=network.name, @@ -949,6 +953,7 @@ def __init__( image_name="system_tests/weblog", name="weblog", environment=environment, + extra_hosts=extra_hosts_for_environment(environment), volumes=volumes, # ddprof's perf event open is blocked by default by docker's seccomp profile # This is worse than the line above though prevents mmap bugs locally diff --git a/utils/_context/weblog_infrastructure.py b/utils/_context/weblog_infrastructure.py index 0a99f700c6b..3796727dff3 100644 --- a/utils/_context/weblog_infrastructure.py +++ b/utils/_context/weblog_infrastructure.py @@ -133,13 +133,14 @@ def _configure_proxy_weblog(self, weblog: str) -> None: self._proxy_runtime_container.depends_on = [self._processor_container, self._dummy_server_container] def set_weblog_dependencies( - self, agent_container: TestedContainer, proxy_container: TestedContainer | None + self, agent_container: TestedContainer | None, proxy_container: TestedContainer | None ) -> None: """Wire container start-order dependencies for all weblog containers. Handles both the standard weblog topology and the go-proxies one transparently. """ - self.library_container.depends_on.append(agent_container) + if agent_container is not None: + self.library_container.depends_on.append(agent_container) if proxy_container is not None: self.library_container.depends_on.append(proxy_container) diff --git a/utils/build/docker/otel_collector/weblog_metadata.yml b/utils/build/docker/otel_collector/weblog_metadata.yml index 39a1eb07ce6..a92ab421b46 100644 --- a/utils/build/docker/otel_collector/weblog_metadata.yml +++ b/utils/build/docker/otel_collector/weblog_metadata.yml @@ -3,4 +3,3 @@ otel_collector: build_mode: none supported_scenarios: [OTEL_COLLECTOR, OTEL_COLLECTOR_E2E] - diff --git a/utils/docker_fixtures/_core.py b/utils/docker_fixtures/_core.py index 3cfacedd571..ddda56f4ce0 100644 --- a/utils/docker_fixtures/_core.py +++ b/utils/docker_fixtures/_core.py @@ -1,5 +1,5 @@ import contextlib -from collections.abc import Generator +from collections.abc import Generator, Mapping from pathlib import Path from typing import TextIO from urllib.parse import urlparse @@ -44,7 +44,7 @@ def compute_volumes(volumes: dict[str, str]) -> dict[str, dict]: return fixed_volumes -def extra_hosts_for_environment(env: dict[str, str]) -> dict[str, str] | None: +def extra_hosts_for_environment(env: Mapping[str, str | None]) -> dict[str, str] | None: for value in env.values(): if urlparse(str(value)).hostname == HOST_DOCKER_INTERNAL: return dict(HOST_GATEWAY_EXTRA_HOSTS) diff --git a/utils/mocked_backend/__init__.py b/utils/mocked_backend/__init__.py new file mode 100644 index 00000000000..272b27ab808 --- /dev/null +++ b/utils/mocked_backend/__init__.py @@ -0,0 +1 @@ +"""Reusable mocked backend implementations for system-test scenarios.""" diff --git a/utils/docker_fixtures/_mock_ffe_agentless_backend.py b/utils/mocked_backend/ffe.py similarity index 99% rename from utils/docker_fixtures/_mock_ffe_agentless_backend.py rename to utils/mocked_backend/ffe.py index 8a77d0eb33b..ac5e028400d 100644 --- a/utils/docker_fixtures/_mock_ffe_agentless_backend.py +++ b/utils/mocked_backend/ffe.py @@ -292,7 +292,7 @@ def _strip_config_path(url: str) -> str: class MockFFEAgentlessBackendServer: - def __init__(self, worker_id: str, *, port: int | None = None) -> None: + def __init__(self, worker_id: str = "master", *, port: int | None = None) -> None: self.port = get_host_port(worker_id, 4900) if port is None else port self._server = MockFFEAgentlessBackendHTTPServer(("0.0.0.0", self.port)) # noqa: S104 - test fixture must be container-reachable. self.port = self._server.server_port diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index 44ebdc207cd..091da60a887 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -224,6 +224,7 @@ patterns: scenario_groups: appsec_rasp_scenario utils/_context/_scenarios/endtoend.py: scenario_groups: end_to_end + scenarios: FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS utils/_context/_scenarios/integration_frameworks.py: scenarios: INTEGRATION_FRAMEWORKS utils/_context/_scenarios/integrations.py: @@ -289,6 +290,9 @@ patterns: utils/interfaces/_library/*: scenario_groups: [end_to_end, open_telemetry] + utils/mocked_backend/ffe.py: + scenarios: [FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS, PARAMETRIC] + utils/_context/virtual_machine.py: scenario_groups: onboarding