From 57f6a99d8babf28b5a2f8cd792741250d3e0d994 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 3 Aug 2026 12:08:35 -0400 Subject: [PATCH 1/4] Add DEFAULT_V1 scenario for Java and Go --- .github/workflows/run-end-to-end.yml | 7 +++-- conftest.py | 3 +- tests/test_the_test/test_ci_orchestrator.py | 30 ++++++++++++++++++- utils/_context/_scenarios/__init__.py | 5 ++++ utils/_context/_scenarios/core.py | 6 ++++ utils/build/docker/golang/weblog_metadata.yml | 6 ++-- utils/build/docker/java/weblog_metadata.yml | 2 +- .../scripts/ci_orchestrators/workflow_data.py | 7 ++++- 8 files changed, 57 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 8d9b8df0aae..5b4ef9427ec 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -112,7 +112,7 @@ jobs: SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: ${{ inputs.skip_empty_scenarios }} SYSTEM_TESTS_FORCE_EXECUTE: ${{ inputs.force_execute }} SYSTEM_TESTS_DEV_MODE: ${{ inputs._system_tests_dev_mode }} - SYSTEM_TESTS_WEBLOG: ${{ inputs.weblog }} + SYSTEM_TESTS_WEBLOG: ${{ inputs.weblog }} steps: - name: Compute ref id: compute_ref @@ -132,7 +132,7 @@ jobs: - name: Install runner uses: ./.github/actions/install_runner - name: Get binaries artifact - if : ${{ inputs.binaries_artifact != '' }} + if: ${{ inputs.binaries_artifact != '' }} uses: ./.github/actions/download_artifact with: name: ${{ inputs.binaries_artifact }} @@ -224,6 +224,9 @@ jobs: - name: Run DEFAULT scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"DEFAULT"') run: ./run.sh DEFAULT + - name: Run DEFAULT_V1 scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"DEFAULT_V1"') + run: ./run.sh DEFAULT_V1 - name: Run GRAPHQL_APPSEC scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"GRAPHQL_APPSEC"') run: ./run.sh GRAPHQL_APPSEC diff --git a/conftest.py b/conftest.py index 8a4047ef975..8f82df66212 100644 --- a/conftest.py +++ b/conftest.py @@ -356,7 +356,8 @@ def iter_markers(self: pytest.Item, name: str | None = None): deselected.append(item) continue - if context.scenario.name in declared_scenarios: + test_scenario_name = "DEFAULT" if context.scenario.name == "DEFAULT_V1" else context.scenario.name + if test_scenario_name in declared_scenarios: logger.info(f"{item.nodeid} is included in {context.scenario}") selected.append(item) diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 2add4b932c7..a687324ffd6 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -6,6 +6,7 @@ from utils._context.weblog_metadata import WeblogMetaData from utils._context._scenarios import get_all_scenarios, Scenario from utils.scripts.ci_orchestrators.workflow_data import ( + _filter_scenarios, _get_endtoend_weblogs, get_endtoend_definitions, ) @@ -51,6 +52,27 @@ def _is_supported(weblog: WeblogMetaData, scenario: Scenario) -> bool: assert _is_supported(get_weblog("python", "flask-poc"), scenarios.ipv6) +# TODO: Remove this test once v1 becomes the global default. +@scenarios.test_the_test +def test_default_v1_uses_java_and_go_default_weblogs(): + assert scenarios.default_v1.weblog_libraries == {"java", "golang"} + assert scenarios.default_v1.weblog_container.environment["DD_TRACE_AGENT_PROTOCOL_VERSION"] == "1.0" + assert scenarios.default_v1.agent_container.environment["DD_APM_ENABLE_V1_TRACE_ENDPOINT"] == "true" + + def supporting_weblogs(scenario: Scenario) -> set[tuple[str, str]]: + return { + (library, weblog.name) + for library in sorted(COMPONENT_GROUPS.all) + for weblog in WeblogMetaData.load(library) + if scenario in _filter_scenarios([scenario], weblog) + } + + default_weblogs = supporting_weblogs(scenarios.default) + expected_v1_weblogs = {weblog for weblog in default_weblogs if weblog[0] in {"java", "golang"}} + + assert supporting_weblogs(scenarios.default_v1) == expected_v1_weblogs + + @scenarios.test_the_test def test_get_endtoend_definitions_empty_scenario_map(): # Regression: previously raised KeyError when "endtoend" or "parametric" keys were absent @@ -153,7 +175,7 @@ def test_legacy_scenario_matrix(): for weblog in sorted(WeblogMetaData.load(library), key=lambda w: w.name): for scenario in get_all_scenarios(): legacy = _is_supported_legacy(weblog, scenario, "") - new_value = weblog.support_scenario(scenario.name, scenario.weblog_categories) + new_value = scenario in _filter_scenarios([scenario], weblog) if legacy is not new_value: has_error = True logger.error((library, legacy, new_value, weblog.name, scenario.name, scenario.weblog_categories)) @@ -206,6 +228,12 @@ def _is_uds_weblog(weblog: str) -> bool: library = weblog.library weblog_name = weblog.name + if not scenario.supports_library(library): + return False + + if scenario.name == "DEFAULT_V1": + scenario = scenarios.default + if library == "c": return scenario.name in ("DEFAULT", "IPV6", "SAMPLING") diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 3efab3e8c48..5fbcb7c6595 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -57,6 +57,11 @@ class _Scenarios: mock_the_test_2 = TestTheTestScenario("MOCK_THE_TEST_2", doc="Mock scenario that check system-tests internals") default = DefaultScenario("DEFAULT") + default_v1 = DefaultScenario("DEFAULT_V1") + # The v1 protocol is under development for other tracers. + default_v1.weblog_libraries = {"java", "golang"} + default_v1.weblog_container.environment["DD_TRACE_AGENT_PROTOCOL_VERSION"] = "1.0" + default_v1.agent_container.environment["DD_APM_ENABLE_V1_TRACE_ENDPOINT"] = "true" integrations = IntegrationsScenario() integrations_aws = AWSIntegrationsScenario("INTEGRATIONS_AWS") diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index ad8656b0aa9..da8edacbb44 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -127,6 +127,9 @@ def __init__( self.weblog_categories: list[WeblogCategory] = weblog_categories or [] """ Which weblog categories should this scenario applies """ + self.weblog_libraries: set[str] | None = None + """Libraries for which this scenario can be scheduled, or all libraries when unset.""" + self.components: dict[str, Version] = {} """ Key-value pair of what is actually tested """ @@ -144,6 +147,9 @@ def __init__( self.warmups: list[Callable] = [] self.collect_only: bool = False + def supports_library(self, library: str) -> bool: + return self.weblog_libraries is None or library in self.weblog_libraries + def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = False): if self.replay: return diff --git a/utils/build/docker/golang/weblog_metadata.yml b/utils/build/docker/golang/weblog_metadata.yml index 0a354c0cee2..a4aec99c400 100644 --- a/utils/build/docker/golang/weblog_metadata.yml +++ b/utils/build/docker/golang/weblog_metadata.yml @@ -2,10 +2,10 @@ haproxy: build_mode: none - supported_scenarios: [APPSEC_BLOCKING, DEFAULT] + supported_scenarios: [APPSEC_BLOCKING, DEFAULT, DEFAULT_V1] envoy: build_mode: none - supported_scenarios: [APPSEC_BLOCKING, DEFAULT] + supported_scenarios: [APPSEC_BLOCKING, DEFAULT, DEFAULT_V1] graphql-go: categories: [dd_trace_graphql] excluded_scenarios: [GRAPHQL_ERROR_TRACKING] # why ??? @@ -29,4 +29,4 @@ chi: net-http-orchestrion: categories: [dd_trace] net-http-span-pool: - categories: [dd_trace] \ No newline at end of file + categories: [dd_trace] diff --git a/utils/build/docker/java/weblog_metadata.yml b/utils/build/docker/java/weblog_metadata.yml index 06f2ce335e0..aab68288d0c 100644 --- a/utils/build/docker/java/weblog_metadata.yml +++ b/utils/build/docker/java/weblog_metadata.yml @@ -35,4 +35,4 @@ uds-spring-boot: vertx3: categories: [dd_trace] vertx4: - categories: [dd_trace] \ No newline at end of file + categories: [dd_trace] diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index af54963fe2d..60c0c249d53 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -490,7 +490,12 @@ def _get_execution_time(library: str, weblog: str, scenario: str, run_stats: dic def _filter_scenarios(scenarios: list[Scenario], weblog: Weblog) -> list[Scenario]: return sorted( - [scenario for scenario in set(scenarios) if weblog.support_scenario(scenario.name, scenario.weblog_categories)], + [ + scenario + for scenario in set(scenarios) + if scenario.supports_library(weblog.library) + and weblog.support_scenario(scenario.name, scenario.weblog_categories) + ], key=lambda scenario: scenario.name, ) From 27473addd7587d53c171c3decf8f64d2190d9350 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 3 Aug 2026 14:43:01 -0400 Subject: [PATCH 2/4] Simplification, keep only tracers that supports v1, but not by default. --- conftest.py | 1 + tests/test_the_test/test_ci_orchestrator.py | 34 +++---------------- utils/_context/_scenarios/__init__.py | 7 ++-- utils/_context/_scenarios/core.py | 8 ++--- utils/build/docker/golang/weblog_metadata.yml | 6 ++-- utils/build/docker/java/weblog_metadata.yml | 2 +- .../scripts/ci_orchestrators/workflow_data.py | 2 +- 7 files changed, 20 insertions(+), 40 deletions(-) diff --git a/conftest.py b/conftest.py index 8f82df66212..9ade0e72e60 100644 --- a/conftest.py +++ b/conftest.py @@ -356,6 +356,7 @@ def iter_markers(self: pytest.Item, name: str | None = None): deselected.append(item) continue + # DEFAULT_V1 uses the same test selection as DEFAULT. test_scenario_name = "DEFAULT" if context.scenario.name == "DEFAULT_V1" else context.scenario.name if test_scenario_name in declared_scenarios: logger.info(f"{item.nodeid} is included in {context.scenario}") diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index a687324ffd6..6fb13a8caf9 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -6,7 +6,6 @@ from utils._context.weblog_metadata import WeblogMetaData from utils._context._scenarios import get_all_scenarios, Scenario from utils.scripts.ci_orchestrators.workflow_data import ( - _filter_scenarios, _get_endtoend_weblogs, get_endtoend_definitions, ) @@ -52,27 +51,6 @@ def _is_supported(weblog: WeblogMetaData, scenario: Scenario) -> bool: assert _is_supported(get_weblog("python", "flask-poc"), scenarios.ipv6) -# TODO: Remove this test once v1 becomes the global default. -@scenarios.test_the_test -def test_default_v1_uses_java_and_go_default_weblogs(): - assert scenarios.default_v1.weblog_libraries == {"java", "golang"} - assert scenarios.default_v1.weblog_container.environment["DD_TRACE_AGENT_PROTOCOL_VERSION"] == "1.0" - assert scenarios.default_v1.agent_container.environment["DD_APM_ENABLE_V1_TRACE_ENDPOINT"] == "true" - - def supporting_weblogs(scenario: Scenario) -> set[tuple[str, str]]: - return { - (library, weblog.name) - for library in sorted(COMPONENT_GROUPS.all) - for weblog in WeblogMetaData.load(library) - if scenario in _filter_scenarios([scenario], weblog) - } - - default_weblogs = supporting_weblogs(scenarios.default) - expected_v1_weblogs = {weblog for weblog in default_weblogs if weblog[0] in {"java", "golang"}} - - assert supporting_weblogs(scenarios.default_v1) == expected_v1_weblogs - - @scenarios.test_the_test def test_get_endtoend_definitions_empty_scenario_map(): # Regression: previously raised KeyError when "endtoend" or "parametric" keys were absent @@ -174,8 +152,12 @@ def test_legacy_scenario_matrix(): for library in sorted(COMPONENT_GROUPS.all): for weblog in sorted(WeblogMetaData.load(library), key=lambda w: w.name): for scenario in get_all_scenarios(): + # DEFAULT_V1 has no legacy equivalent. + if scenario is scenarios.default_v1: + continue + legacy = _is_supported_legacy(weblog, scenario, "") - new_value = scenario in _filter_scenarios([scenario], weblog) + new_value = weblog.support_scenario(scenario.name, scenario.weblog_categories) if legacy is not new_value: has_error = True logger.error((library, legacy, new_value, weblog.name, scenario.name, scenario.weblog_categories)) @@ -228,12 +210,6 @@ def _is_uds_weblog(weblog: str) -> bool: library = weblog.library weblog_name = weblog.name - if not scenario.supports_library(library): - return False - - if scenario.name == "DEFAULT_V1": - scenario = scenarios.default - if library == "c": return scenario.name in ("DEFAULT", "IPV6", "SAMPLING") diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 5fbcb7c6595..89a62eb2046 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -57,9 +57,12 @@ class _Scenarios: mock_the_test_2 = TestTheTestScenario("MOCK_THE_TEST_2", doc="Mock scenario that check system-tests internals") default = DefaultScenario("DEFAULT") + + # Run the DEFAULT suite with the v1 protocol forced. + # Go uses v1 by default, while Java supports v1 but defaults to v0.4. + # Keep only tracers that support v1 but do not yet use it by default. default_v1 = DefaultScenario("DEFAULT_V1") - # The v1 protocol is under development for other tracers. - default_v1.weblog_libraries = {"java", "golang"} + default_v1.supported_tracers = {"java"} default_v1.weblog_container.environment["DD_TRACE_AGENT_PROTOCOL_VERSION"] = "1.0" default_v1.agent_container.environment["DD_APM_ENABLE_V1_TRACE_ENDPOINT"] = "true" diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index da8edacbb44..f37a6d3796e 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -127,8 +127,8 @@ def __init__( self.weblog_categories: list[WeblogCategory] = weblog_categories or [] """ Which weblog categories should this scenario applies """ - self.weblog_libraries: set[str] | None = None - """Libraries for which this scenario can be scheduled, or all libraries when unset.""" + self.supported_tracers: set[str] | None = None + """Tracers for which this scenario can be scheduled, or all tracers when unset.""" self.components: dict[str, Version] = {} """ Key-value pair of what is actually tested """ @@ -147,8 +147,8 @@ def __init__( self.warmups: list[Callable] = [] self.collect_only: bool = False - def supports_library(self, library: str) -> bool: - return self.weblog_libraries is None or library in self.weblog_libraries + def supports_tracer(self, tracer: str) -> bool: + return self.supported_tracers is None or tracer in self.supported_tracers def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = False): if self.replay: diff --git a/utils/build/docker/golang/weblog_metadata.yml b/utils/build/docker/golang/weblog_metadata.yml index a4aec99c400..0a354c0cee2 100644 --- a/utils/build/docker/golang/weblog_metadata.yml +++ b/utils/build/docker/golang/weblog_metadata.yml @@ -2,10 +2,10 @@ haproxy: build_mode: none - supported_scenarios: [APPSEC_BLOCKING, DEFAULT, DEFAULT_V1] + supported_scenarios: [APPSEC_BLOCKING, DEFAULT] envoy: build_mode: none - supported_scenarios: [APPSEC_BLOCKING, DEFAULT, DEFAULT_V1] + supported_scenarios: [APPSEC_BLOCKING, DEFAULT] graphql-go: categories: [dd_trace_graphql] excluded_scenarios: [GRAPHQL_ERROR_TRACKING] # why ??? @@ -29,4 +29,4 @@ chi: net-http-orchestrion: categories: [dd_trace] net-http-span-pool: - categories: [dd_trace] + categories: [dd_trace] \ No newline at end of file diff --git a/utils/build/docker/java/weblog_metadata.yml b/utils/build/docker/java/weblog_metadata.yml index aab68288d0c..06f2ce335e0 100644 --- a/utils/build/docker/java/weblog_metadata.yml +++ b/utils/build/docker/java/weblog_metadata.yml @@ -35,4 +35,4 @@ uds-spring-boot: vertx3: categories: [dd_trace] vertx4: - categories: [dd_trace] + categories: [dd_trace] \ No newline at end of file diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 60c0c249d53..4ddebd1ca8c 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -493,7 +493,7 @@ def _filter_scenarios(scenarios: list[Scenario], weblog: Weblog) -> list[Scenari [ scenario for scenario in set(scenarios) - if scenario.supports_library(weblog.library) + if scenario.supports_tracer(weblog.library) and weblog.support_scenario(scenario.name, scenario.weblog_categories) ], key=lambda scenario: scenario.name, From ef40c248f883202b688d20babb9ee01e69094b31 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 5 Aug 2026 15:50:34 -0400 Subject: [PATCH 3/4] Refine Java v1 protocol test coverage --- .github/workflows/run-end-to-end.yml | 13 ++-- .github/workflows/system-tests.yml | 1 + conftest.py | 4 +- tests/test_the_test/test_ci_orchestrator.py | 34 +++++++-- tests/test_the_test/test_docker_scenario.py | 13 +++- utils/_context/_scenarios/__init__.py | 11 --- utils/_context/_scenarios/core.py | 6 -- utils/_context/_scenarios/endtoend.py | 10 +++ .../scripts/ci_orchestrators/workflow_data.py | 72 +++++++++++++++++-- 9 files changed, 127 insertions(+), 37 deletions(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 5b4ef9427ec..7a9783ff2be 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -101,6 +101,11 @@ on: default: false required: false type: boolean + weblog_env: + description: "JSON object of environment variables to force on the weblog" + default: "{}" + required: false + type: string jobs: main: @@ -112,7 +117,8 @@ jobs: SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: ${{ inputs.skip_empty_scenarios }} SYSTEM_TESTS_FORCE_EXECUTE: ${{ inputs.force_execute }} SYSTEM_TESTS_DEV_MODE: ${{ inputs._system_tests_dev_mode }} - SYSTEM_TESTS_WEBLOG: ${{ inputs.weblog }} + SYSTEM_TESTS_WEBLOG: ${{ inputs.weblog }} + SYSTEM_TESTS_WEBLOG_ENV: ${{ inputs.weblog_env }} steps: - name: Compute ref id: compute_ref @@ -132,7 +138,7 @@ jobs: - name: Install runner uses: ./.github/actions/install_runner - name: Get binaries artifact - if: ${{ inputs.binaries_artifact != '' }} + if : ${{ inputs.binaries_artifact != '' }} uses: ./.github/actions/download_artifact with: name: ${{ inputs.binaries_artifact }} @@ -224,9 +230,6 @@ jobs: - name: Run DEFAULT scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"DEFAULT"') run: ./run.sh DEFAULT - - name: Run DEFAULT_V1 scenario - if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"DEFAULT_V1"') - run: ./run.sh DEFAULT_V1 - name: Run GRAPHQL_APPSEC scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"GRAPHQL_APPSEC"') run: ./run.sh GRAPHQL_APPSEC diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index ffe17ac2c21..ec45b71d2e0 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -263,6 +263,7 @@ jobs: weblog_instance: ${{ matrix.job.weblog_instance }} weblog_build_required: ${{ matrix.job.weblog_build_required }} scenarios: ${{ toJson(matrix.job.scenarios) }} + weblog_env: ${{ toJson(matrix.job.weblog_env) }} force_execute: ${{ inputs.force_execute }} binaries_artifact: ${{ matrix.job.binaries_artifact }} ci_environment: ${{ needs.compute_parameters.outputs.ci_environment }} diff --git a/conftest.py b/conftest.py index 9ade0e72e60..8a4047ef975 100644 --- a/conftest.py +++ b/conftest.py @@ -356,9 +356,7 @@ def iter_markers(self: pytest.Item, name: str | None = None): deselected.append(item) continue - # DEFAULT_V1 uses the same test selection as DEFAULT. - test_scenario_name = "DEFAULT" if context.scenario.name == "DEFAULT_V1" else context.scenario.name - if test_scenario_name in declared_scenarios: + if context.scenario.name in declared_scenarios: logger.info(f"{item.nodeid} is included in {context.scenario}") selected.append(item) diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 6fb13a8caf9..2597269db62 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -6,6 +6,8 @@ from utils._context.weblog_metadata import WeblogMetaData from utils._context._scenarios import get_all_scenarios, Scenario from utils.scripts.ci_orchestrators.workflow_data import ( + Job, + _add_java_v1_protocol_jobs, _get_endtoend_weblogs, get_endtoend_definitions, ) @@ -141,10 +143,38 @@ def test_otel_collector(): "weblog": "otel_collector", "weblog_build_required": False, "weblog_instance": 1, + "weblog_env": {}, } ] +@scenarios.test_the_test +def test_java_prod_v1_protocol_jobs(): + weblog = get_weblog("java", "spring-boot-jetty") + other_weblog = get_weblog("java", "spring-boot") + jobs = [ + Job("java", weblog, 1, {"DEFAULT": 10.0, "FIRST": 20.0}, 5.0, build_base_images=False), + Job("java", weblog, 2, {"SECOND": 30.0}, 5.0, build_base_images=False), + Job("java", weblog, 3, {"THIRD": 40.0}, 5.0, build_base_images=False), + Job("java", other_weblog, 1, {"DEFAULT": 10.0, "OTHER": 20.0}, 5.0, build_base_images=False), + ] + + result = _add_java_v1_protocol_jobs(jobs, library="java", ci_environment="prod") + v1_jobs = {f"{job.weblog.name} {job.serialize()['weblog_instance']}": job for job in result} + + assert set(v1_jobs) == { + "spring-boot-jetty 1_v1", + "spring-boot-jetty 2_v1", + "spring-boot-jetty 3_v1", + "spring-boot 1_v1", + } + assert v1_jobs["spring-boot-jetty 1_v1"].scenarios == ("DEFAULT", "FIRST") + assert v1_jobs["spring-boot 1_v1"].scenarios == ("DEFAULT",) + assert all(job.weblog_env == {"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"} for job in v1_jobs.values()) + assert _add_java_v1_protocol_jobs(jobs, library="java", ci_environment="dev") == [] + assert _add_java_v1_protocol_jobs(jobs, library="nodejs", ci_environment="prod") == [] + + @scenarios.test_the_test def test_legacy_scenario_matrix(): has_error = False @@ -152,10 +182,6 @@ def test_legacy_scenario_matrix(): for library in sorted(COMPONENT_GROUPS.all): for weblog in sorted(WeblogMetaData.load(library), key=lambda w: w.name): for scenario in get_all_scenarios(): - # DEFAULT_V1 has no legacy equivalent. - if scenario is scenarios.default_v1: - continue - legacy = _is_supported_legacy(weblog, scenario, "") new_value = weblog.support_scenario(scenario.name, scenario.weblog_categories) if legacy is not new_value: diff --git a/tests/test_the_test/test_docker_scenario.py b/tests/test_the_test/test_docker_scenario.py index 51e0cebf268..42f58aa25cc 100644 --- a/tests/test_the_test/test_docker_scenario.py +++ b/tests/test_the_test/test_docker_scenario.py @@ -4,7 +4,7 @@ import pytest from utils import interfaces, scenarios -from utils._context._scenarios.endtoend import DdTraceEndToEndScenario, DockerScenario +from utils._context._scenarios.endtoend import DdTraceEndToEndScenario, DockerScenario, _load_environment_overrides from utils._context.containers import TestedContainer as _TestedContainer @@ -24,6 +24,17 @@ def remove(self): pass +@scenarios.test_the_test +def test_load_environment_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SYSTEM_TESTS_WEBLOG_ENV", '{"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"}') + + assert _load_environment_overrides("SYSTEM_TESTS_WEBLOG_ENV") == {"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"} + + monkeypatch.setenv("SYSTEM_TESTS_WEBLOG_ENV", '["not", "an", "object"]') + with pytest.raises(ValueError, match="must be a JSON object"): + _load_environment_overrides("SYSTEM_TESTS_WEBLOG_ENV") + + @scenarios.test_the_test def test_main(): events: list[str] = [] diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index f02d1e7bea5..0c6e8e0601a 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -58,14 +58,6 @@ class _Scenarios: default = DefaultScenario("DEFAULT") - # Run the DEFAULT suite with the v1 protocol forced. - # Go uses v1 by default, while Java supports v1 but defaults to v0.4. - # Keep only tracers that support v1 but do not yet use it by default. - default_v1 = DefaultScenario("DEFAULT_V1") - default_v1.supported_tracers = {"java"} - default_v1.weblog_container.environment["DD_TRACE_AGENT_PROTOCOL_VERSION"] = "1.0" - default_v1.agent_container.environment["DD_APM_ENABLE_V1_TRACE_ENDPOINT"] = "true" - integrations = IntegrationsScenario() integrations_aws = AWSIntegrationsScenario("INTEGRATIONS_AWS") dbm_dynamic_service = DbmDynamicServiceScenario() @@ -848,9 +840,6 @@ class _Scenarios: "DD_TRACE_SAMPLE_RATE": "1.0", "DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0", }, - agent_env={ - "DD_APM_ENABLE_V1_TRACE_ENDPOINT": "true", - }, backend_interface_timeout=5, doc="End-to-end testing scenario focused on efficient payload handling and v1 trace format validation", ) diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index f37a6d3796e..ad8656b0aa9 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -127,9 +127,6 @@ def __init__( self.weblog_categories: list[WeblogCategory] = weblog_categories or [] """ Which weblog categories should this scenario applies """ - self.supported_tracers: set[str] | None = None - """Tracers for which this scenario can be scheduled, or all tracers when unset.""" - self.components: dict[str, Version] = {} """ Key-value pair of what is actually tested """ @@ -147,9 +144,6 @@ def __init__( self.warmups: list[Callable] = [] self.collect_only: bool = False - def supports_tracer(self, tracer: str) -> bool: - return self.supported_tracers is None or tracer in self.supported_tracers - def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = False): if self.replay: return diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index a213b207620..b2649a28c51 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -37,6 +37,15 @@ from .core import Scenario, ScenarioGroup, scenario_groups as all_scenario_groups +def _load_environment_overrides(variable_name: str) -> dict[str, str]: + value: object = json.loads(os.environ.get(variable_name, "{}")) + if not isinstance(value, dict) or not all( + isinstance(key, str) and isinstance(item, str) for key, item in value.items() + ): + raise ValueError(f"{variable_name} must be a JSON object with string keys and values") + return value + + class DockerScenario(Scenario): """Scenario that tests docker containers""" @@ -328,6 +337,7 @@ def configure(self, config: pytest.Config): pytest.exit("DD_API_KEY is required for this scenario", 1) self.weblog_infra.configure(config) + self.weblog_infra.library_container.environment.update(_load_environment_overrides("SYSTEM_TESTS_WEBLOG_ENV")) self._containers += list(self.weblog_infra.get_containers()) self._set_containers_dependancies() diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4ddebd1ca8c..fa7c19efd12 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -200,6 +200,8 @@ def get_docker_ssi_matrix( # End-to-end corner +_V1_PROTOCOL_WEBLOG_ENV = {"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"} + class Job: """a job is a couple weblog/scenarios that will be executed in a single runner""" @@ -213,6 +215,8 @@ def __init__( build_time: float, *, build_base_images: bool, + name_suffix: str = "", + weblog_env: dict[str, str] | None = None, ): self.library = library self.weblog = weblog @@ -223,6 +227,9 @@ def __init__( # as a given weblog can have multiple runner executing its scenarios # weblog_instance will be used to differentiate them self.weblog_instance = weblog_instance + self.name_suffix = name_suffix + + self.weblog_env = weblog_env or {} # build_time is not directly tight to the job, as another runner will execute it # but it's convenient to store this info here, as we'll need it to execute the @@ -238,8 +245,11 @@ def serialize(self) -> dict: "library": self.library, "weblog": self.weblog.name, "weblog_build_required": self.weblog.require_build, - "weblog_instance": self.weblog_instance, + "weblog_instance": f"{self.weblog_instance}{self.name_suffix}" + if self.name_suffix + else self.weblog_instance, "scenarios": sorted(self.scenarios), + "weblog_env": self.weblog_env, "expected_job_time": self.expected_job_time + self.build_time, "binaries_artifact": self.weblog.artifact_name, "build_weblog_base_image": self.weblog.build_mode == BuildMode.local @@ -257,6 +267,10 @@ def expected_job_time(self) -> float: @property def sort_key(self) -> tuple: + return (self.weblog.name, self.weblog_instance, self.name_suffix) + + @property + def identity(self) -> tuple[str, int]: return (self.weblog.name, self.weblog_instance) def get_scenario_time(self, scenario: str) -> float: @@ -266,6 +280,27 @@ def append_scenario(self, scenario: str, execution_time: float) -> None: assert scenario not in self._scenarios_times self._scenarios_times[scenario] = execution_time + def duplicate( + self, + *, + name_suffix: str, + scenarios: tuple[str, ...] | None = None, + weblog_env: dict[str, str] | None = None, + ) -> "Job": + selected_scenarios = scenarios or self.scenarios + assert set(selected_scenarios) <= set(self.scenarios) + + return Job( + library=self.library, + weblog=self.weblog, + weblog_instance=self.weblog_instance, + scenarios_times={scenario: self._scenarios_times[scenario] for scenario in selected_scenarios}, + build_time=self.build_time, + build_base_images=self.build_base_images, + name_suffix=name_suffix, + weblog_env=weblog_env, + ) + def split_for_parallel_execution(self, desired_execution_time: float) -> list["Job"]: result: list[Job] = [] @@ -371,6 +406,8 @@ def get_endtoend_definitions( if desired_execution_time > 0: # 0 or less means that user doesn't want to split jobs jobs = _split_jobs_for_parallel_execution(jobs, desired_execution_time, maximum_parallel_jobs) + jobs.extend(_add_java_v1_protocol_jobs(jobs, library=library, ci_environment=ci_environment)) + # sort jobs by weblog name and weblog instance jobs.sort(key=lambda job: job.sort_key) @@ -427,6 +464,32 @@ def _split_jobs_for_parallel_execution( return result +def _add_java_v1_protocol_jobs(jobs: list[Job], *, library: str, ci_environment: str) -> list[Job]: + if library != "java" or ci_environment != "prod": + return [] + + # Exercise DEFAULT on every Java weblog. For spring-boot-jetty, replace the + # DEFAULT-only duplicate with complete duplicates of its three CI shards. + duplicates = { + job.identity: job.duplicate( + name_suffix="_v1", + scenarios=("DEFAULT",), + weblog_env=_V1_PROTOCOL_WEBLOG_ENV, + ) + for job in jobs + if "DEFAULT" in job.scenarios + } + + for job in jobs: + if job.weblog.name == "spring-boot-jetty" and job.weblog_instance in (1, 2, 3): + duplicates[job.identity] = job.duplicate( + name_suffix="_v1", + weblog_env=_V1_PROTOCOL_WEBLOG_ENV, + ) + + return sorted(duplicates.values(), key=lambda job: job.sort_key) + + def _split_scenarios_for_parallel_execution( scenario_times: dict[str, float], desired_execution_time: float ) -> list[list[str]]: @@ -490,12 +553,7 @@ def _get_execution_time(library: str, weblog: str, scenario: str, run_stats: dic def _filter_scenarios(scenarios: list[Scenario], weblog: Weblog) -> list[Scenario]: return sorted( - [ - scenario - for scenario in set(scenarios) - if scenario.supports_tracer(weblog.library) - and weblog.support_scenario(scenario.name, scenario.weblog_categories) - ], + [scenario for scenario in set(scenarios) if weblog.support_scenario(scenario.name, scenario.weblog_categories)], key=lambda scenario: scenario.name, ) From 4e67249cc6c3491a5285104f6d4ea429754d13d2 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 5 Aug 2026 16:29:33 -0400 Subject: [PATCH 4/4] Refactor protocol job duplication by weblog --- tests/test_the_test/test_ci_orchestrator.py | 27 +++++---- .../scripts/ci_orchestrators/workflow_data.py | 57 ++++++++++--------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 2597269db62..bb995bec14f 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -7,7 +7,7 @@ from utils._context._scenarios import get_all_scenarios, Scenario from utils.scripts.ci_orchestrators.workflow_data import ( Job, - _add_java_v1_protocol_jobs, + _duplicate_jobs, _get_endtoend_weblogs, get_endtoend_definitions, ) @@ -149,30 +149,33 @@ def test_otel_collector(): @scenarios.test_the_test -def test_java_prod_v1_protocol_jobs(): +def test_duplicate_jobs_for_selected_weblogs(): weblog = get_weblog("java", "spring-boot-jetty") other_weblog = get_weblog("java", "spring-boot") jobs = [ - Job("java", weblog, 1, {"DEFAULT": 10.0, "FIRST": 20.0}, 5.0, build_base_images=False), - Job("java", weblog, 2, {"SECOND": 30.0}, 5.0, build_base_images=False), - Job("java", weblog, 3, {"THIRD": 40.0}, 5.0, build_base_images=False), - Job("java", other_weblog, 1, {"DEFAULT": 10.0, "OTHER": 20.0}, 5.0, build_base_images=False), + Job("java", weblog, 1, {"DEFAULT": 1.0, "JETTY_SCENARIO_1": 1.0}, 1.0, build_base_images=False), + Job("java", weblog, 2, {"JETTY_SCENARIO_2": 1.0}, 1.0, build_base_images=False), + Job("java", weblog, 3, {"JETTY_SCENARIO_3": 1.0}, 1.0, build_base_images=False), + Job("java", other_weblog, 1, {"DEFAULT": 1.0, "NON_DEFAULT_SCENARIO": 1.0}, 1.0, build_base_images=False), ] - result = _add_java_v1_protocol_jobs(jobs, library="java", ci_environment="prod") + result = _duplicate_jobs( + jobs, + weblog_names=("spring-boot-jetty",), + name_suffix="_v1", + weblog_env={"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"}, + ) v1_jobs = {f"{job.weblog.name} {job.serialize()['weblog_instance']}": job for job in result} assert set(v1_jobs) == { "spring-boot-jetty 1_v1", "spring-boot-jetty 2_v1", "spring-boot-jetty 3_v1", - "spring-boot 1_v1", } - assert v1_jobs["spring-boot-jetty 1_v1"].scenarios == ("DEFAULT", "FIRST") - assert v1_jobs["spring-boot 1_v1"].scenarios == ("DEFAULT",) + assert v1_jobs["spring-boot-jetty 1_v1"].scenarios == ("DEFAULT", "JETTY_SCENARIO_1") + assert v1_jobs["spring-boot-jetty 2_v1"].scenarios == ("JETTY_SCENARIO_2",) + assert v1_jobs["spring-boot-jetty 3_v1"].scenarios == ("JETTY_SCENARIO_3",) assert all(job.weblog_env == {"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"} for job in v1_jobs.values()) - assert _add_java_v1_protocol_jobs(jobs, library="java", ci_environment="dev") == [] - assert _add_java_v1_protocol_jobs(jobs, library="nodejs", ci_environment="prod") == [] @scenarios.test_the_test diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index fa7c19efd12..99948f73aa3 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -200,8 +200,6 @@ def get_docker_ssi_matrix( # End-to-end corner -_V1_PROTOCOL_WEBLOG_ENV = {"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"} - class Job: """a job is a couple weblog/scenarios that will be executed in a single runner""" @@ -406,7 +404,18 @@ def get_endtoend_definitions( if desired_execution_time > 0: # 0 or less means that user doesn't want to split jobs jobs = _split_jobs_for_parallel_execution(jobs, desired_execution_time, maximum_parallel_jobs) - jobs.extend(_add_java_v1_protocol_jobs(jobs, library=library, ci_environment=ci_environment)) + # Duplicate selected test jobs to exercise an alternative protocol without changing the original jobs. + # This can force v1 while the originals use v0.x, or test a legacy protocol while they use the current one. + # Add, remove, or adjust library-specific selections here as protocol coverage evolves. + if library == "java" and ci_environment == "prod": + jobs.extend( + _duplicate_jobs( + jobs, + weblog_names=("spring-boot-jetty",), + name_suffix="_v1", + weblog_env={"DD_TRACE_AGENT_PROTOCOL_VERSION": "1.0"}, + ) + ) # sort jobs by weblog name and weblog instance jobs.sort(key=lambda job: job.sort_key) @@ -464,30 +473,26 @@ def _split_jobs_for_parallel_execution( return result -def _add_java_v1_protocol_jobs(jobs: list[Job], *, library: str, ci_environment: str) -> list[Job]: - if library != "java" or ci_environment != "prod": - return [] - - # Exercise DEFAULT on every Java weblog. For spring-boot-jetty, replace the - # DEFAULT-only duplicate with complete duplicates of its three CI shards. - duplicates = { - job.identity: job.duplicate( - name_suffix="_v1", - scenarios=("DEFAULT",), - weblog_env=_V1_PROTOCOL_WEBLOG_ENV, - ) - for job in jobs - if "DEFAULT" in job.scenarios - } - - for job in jobs: - if job.weblog.name == "spring-boot-jetty" and job.weblog_instance in (1, 2, 3): - duplicates[job.identity] = job.duplicate( - name_suffix="_v1", - weblog_env=_V1_PROTOCOL_WEBLOG_ENV, +def _duplicate_jobs( + jobs: list[Job], + *, + weblog_names: tuple[str, ...], + name_suffix: str, + weblog_env: dict[str, str] | None = None, +) -> list[Job]: + """Duplicate every job for the selected weblogs.""" + selected_weblogs = set(weblog_names) + return sorted( + [ + job.duplicate( + name_suffix=name_suffix, + weblog_env=weblog_env, ) - - return sorted(duplicates.values(), key=lambda job: job.sort_key) + for job in jobs + if job.weblog.name in selected_weblogs + ], + key=lambda job: job.sort_key, + ) def _split_scenarios_for_parallel_execution(