diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 8ff3aa9bfc4..97b61f0bd35 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -719,6 +719,9 @@ manifest: component_version: <3.22.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) diff --git a/manifests/golang.yml b/manifests/golang.yml index 89a68ba29e0..f482c1b4d53 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1118,6 +1118,9 @@ manifest: tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0 tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 diff --git a/manifests/java.yml b/manifests/java.yml index 209df5639a6..0a16a87508e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3249,7 +3249,10 @@ manifest: component_version: <1.52.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: 1.61.0-SNAPSHOT # Normalization of telemetry keys updated in https://github.com/DataDog/dd-trace-java/pull/10823 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) - tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_configuration.py: v1.65.0-SNAPSHOT + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 05eb5f818ff..617e271445d 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1706,6 +1706,9 @@ manifest: component_version: <5.66.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: *ref_5_83_0 tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/php.yml b/manifests/php.yml index 26e0dcfe281..600752fd49c 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -710,6 +710,9 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v1.8.3 tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: missing_feature (No implemented the endpoint /crashme) tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index 8b1bf2060cd..b343062c58c 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1344,6 +1344,9 @@ manifest: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 0559dfe53fb..9f31f981f9f 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1930,6 +1930,9 @@ manifest: component_version: <2.7.0 tests/docker_ssi/test_docker_ssi_servicenaming.py::TestDockerServiceNaming::test_service_name: missing_feature (No implemented) tests/ffe/test_agentless_configuration.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_exposures.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_evp.py: missing_feature (agentless end-to-end scenario not enabled yet) + tests/ffe/test_agentless_flag_eval_metrics.py: missing_feature (agentless end-to-end scenario not enabled yet) tests/ffe/test_dynamic_evaluation.py: - weblog_declaration: "*": irrelevant diff --git a/tests/ffe/test_agentless_exposures.py b/tests/ffe/test_agentless_exposures.py new file mode 100644 index 00000000000..d863844f4d3 --- /dev/null +++ b/tests/ffe/test_agentless_exposures.py @@ -0,0 +1,28 @@ +"""Test exposure events when UFC is loaded through the default agentless source.""" + +from tests.ffe.test_exposures import find_exposure_events, wait_for_exposure_event +from tests.ffe.utils.evaluation import evaluate_flag +from utils import features, scenarios + + +@scenarios.feature_flagging_and_experimentation_agentless +@features.feature_flags_exposures +class Test_FFE_Agentless_Exposure: + flag_key = "empty-targeting-key-flag" + targeting_key = "agentless-exposure-user" + + def setup_agentless_exposure(self) -> None: + self.response = evaluate_flag(self.flag_key, targeting_key=self.targeting_key) + + def test_agentless_exposure(self) -> None: + assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" + + wait_for_exposure_event({self.flag_key}, self.targeting_key) + events = find_exposure_events(self.flag_key, self.targeting_key) + assert events, f"No exposure event found for {self.flag_key} and {self.targeting_key}" + + event = events[0] + assert event["flag"]["key"] == self.flag_key + assert event["variant"]["key"] == "on" + assert event["allocation"]["key"] == "default-allocation" + assert event["subject"]["id"] == self.targeting_key diff --git a/tests/ffe/test_agentless_flag_eval_evp.py b/tests/ffe/test_agentless_flag_eval_evp.py new file mode 100644 index 00000000000..1bd1850c88b --- /dev/null +++ b/tests/ffe/test_agentless_flag_eval_evp.py @@ -0,0 +1,36 @@ +"""Test aggregate evaluation EVP when UFC is loaded through the default agentless source.""" + +import pytest + +from tests.ffe.test_flag_eval_evp import ( + assert_batch_context, + assert_event_contract, + find_evp_flagevaluation_events, + object_key, + wait_for_evp_flagevaluation_event, +) +from tests.ffe.utils.evaluation import evaluate_flag +from utils import features, scenarios + + +@scenarios.feature_flagging_and_experimentation_agentless +@features.feature_flags_evp_flagevaluation +@pytest.mark.skip_if_xfail +class Test_FFE_Agentless_EVP_Flagevaluation: + flag_key = "empty-targeting-key-flag" + + def setup_agentless_evp_flagevaluation(self) -> None: + self.response = evaluate_flag(self.flag_key, targeting_key="agentless-evp-user") + + def test_agentless_evp_flagevaluation(self) -> None: + assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" + + wait_for_evp_flagevaluation_event(self.flag_key) + events = find_evp_flagevaluation_events(self.flag_key) + assert events, f"No EVP flagevaluation event found for {self.flag_key}" + + batch, event = events[0] + assert_batch_context(batch) + assert_event_contract(event, self.flag_key) + assert object_key(event.get("variant"), "variant") == "on" + assert object_key(event.get("allocation"), "allocation") == "default-allocation" diff --git a/tests/ffe/test_agentless_flag_eval_metrics.py b/tests/ffe/test_agentless_flag_eval_metrics.py new file mode 100644 index 00000000000..2c7e14bd68d --- /dev/null +++ b/tests/ffe/test_agentless_flag_eval_metrics.py @@ -0,0 +1,25 @@ +"""Test evaluation metrics when UFC is loaded through the default agentless source.""" + +from tests.ffe.test_flag_eval_metrics import find_eval_metrics, get_tag_value +from tests.ffe.utils.evaluation import evaluate_flag +from utils import features, scenarios + + +@scenarios.feature_flagging_and_experimentation_agentless +@features.feature_flags_eval_metrics +class Test_FFE_Agentless_Eval_Metric: + flag_key = "empty-targeting-key-flag" + + def setup_agentless_eval_metric(self) -> None: + self.response = evaluate_flag(self.flag_key, targeting_key="agentless-metric-user") + + def test_agentless_eval_metric(self) -> None: + assert self.response.status_code == 200, f"Flag evaluation failed: {self.response.text}" + + metrics = find_eval_metrics(self.flag_key) + assert metrics, f"No feature_flag.evaluations metric found for {self.flag_key}" + + tags = metrics[0].get("tags", []) + assert get_tag_value(tags, "feature_flag.key") == self.flag_key + assert get_tag_value(tags, "feature_flag.result.variant") == "on" + assert get_tag_value(tags, "feature_flag.result.allocation_key") == "default-allocation" diff --git a/tests/ffe/utils/evaluation.py b/tests/ffe/utils/evaluation.py new file mode 100644 index 00000000000..1c62e5f5018 --- /dev/null +++ b/tests/ffe/utils/evaluation.py @@ -0,0 +1,25 @@ +"""Source-neutral helpers for end-to-end feature flag evaluation.""" + +from utils import HttpResponse, weblog + +from .fixtures import JSON + + +def evaluate_flag( + flag_key: str, + *, + targeting_key: str = "user-1", + attributes: JSON | None = None, + variation_type: str = "STRING", + default_value: object = "default", +) -> HttpResponse: + return weblog.post( + "/ffe", + json={ + "flag": flag_key, + "variationType": variation_type, + "defaultValue": default_value, + "targetingKey": targeting_key, + "attributes": attributes or {}, + }, + )