From 0c72b97bd0fb51409f4e5473c7acf7d115e3e004 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Thu, 4 Jun 2026 18:04:16 -0700 Subject: [PATCH] chore: GenAI Client(evals) - normalize inference model names in create_evaluation_run PiperOrigin-RevId: 926988958 --- agentplatform/_genai/_evals_common.py | 70 +++++++++++++++++- agentplatform/_genai/evals.py | 10 +++ agentplatform/_genai/types/common.py | 4 +- tests/unit/agentplatform/genai/test_evals.py | 75 ++++++++++++++++++++ vertexai/_genai/_evals_common.py | 70 +++++++++++++++++- vertexai/_genai/evals.py | 10 +++ vertexai/_genai/types/common.py | 4 +- 7 files changed, 237 insertions(+), 6 deletions(-) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 8a41bf0a25..09a8ad59cb 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -513,9 +513,21 @@ def _resolve_inference_configs( else: config.agent_configs = parsed_agent_info.agents - # Resolve prompt template data if inference_configs: for inference_config in inference_configs.values(): + model_val = ( + inference_config.get("model") + if isinstance(inference_config, dict) + else inference_config.model + ) + if model_val: + normalized_model = _normalize_inference_model_name( + model_val, api_client + ) + if isinstance(inference_config, dict): + inference_config["model"] = normalized_model + else: + inference_config.model = normalized_model prompt_template_val = ( inference_config.get("prompt_template") if isinstance(inference_config, dict) @@ -912,6 +924,62 @@ def _is_gemini_model(model: str) -> bool: ) +def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str: + """Expands a model name to a fully-qualified resource name for inference. + + A short or location-less model name has no serving location for the + Evaluation Service to route on, so it is expanded using the client's + project and location. Already fully-qualified names pass through. Raises + ValueError if the client is missing a project or location, or if the model + name is not a recognized Vertex form. + """ + if not model: + return model + + if model.startswith("projects/"): + return model + + project = getattr(api_client, "project", None) + location = getattr(api_client, "location", None) + prefix = f"projects/{project}/locations/{location}/" + + def _require_project_location() -> None: + if not project or not location: + raise ValueError( + f"Cannot expand model name '{model}' to a fully-qualified" + " resource name because the client is missing a project or" + " location. Set project and location on the client, or pass a" + " fully-qualified" + " 'projects/{project}/locations/{location}/publishers/google/models/{model}'" + " resource name." + ) + + if ( + model.startswith("publishers/") + or model.startswith("endpoints/") + or model.startswith("tunedModels/") + ): + _require_project_location() + return f"{prefix}{model}" + + if model.startswith("models/"): + _require_project_location() + return f"{prefix}publishers/google/{model}" + + if "/" not in model and _is_gemini_model(model): + _require_project_location() + return f"{prefix}publishers/google/models/{model}" + + raise ValueError( + f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g." + " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint" + " resource name (e.g." + " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'" + " or" + " 'projects/{project}/locations/{location}/endpoints/{endpoint}')." + ) + + def _run_inference_internal( api_client: BaseApiClient, model: Union[Callable[[Any], Any], str], diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index dbbabdd1d7..fd0ae68139 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -2659,6 +2659,11 @@ def create_evaluation_run( The key is the candidate name, and the value is the inference config. If provided, `agent_info` must be None. If omitted and `agent_info` is provided, this will be automatically constructed using `agent_info` and `user_simulator_config`. + The `model` field of an inference config accepts a short Gemini model + name (e.g. `gemini-2.5-flash`), which is automatically expanded to a + fully-qualified resource name using the client's project and location, + or an already fully-qualified publisher-model or endpoint resource + name. Example: {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")} labels: The labels to apply to the evaluation run. @@ -4448,6 +4453,11 @@ async def create_evaluation_run( The key is the candidate name, and the value is the inference config. If provided, `agent_info` must be None. If omitted and `agent_info` is provided, this will be automatically constructed using `agent_info` and `user_simulator_config`. + The `model` field of an inference config accepts a short Gemini model + name (e.g. `gemini-2.5-flash`), which is automatically expanded to a + fully-qualified resource name using the client's project and location, + or an already fully-qualified publisher-model or endpoint resource + name. Example: {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")} red_teaming_config: This field is experimental and may change in future diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 347b269a16..c1351dcce9 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -2586,7 +2586,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel): ) model: Optional[str] = Field( default=None, - description="""The fully qualified name of the publisher model or endpoint to use for inference.""", + description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""", ) prompt_template: Optional[EvaluationRunPromptTemplate] = Field( default=None, description="""The prompt template used for inference.""" @@ -2611,7 +2611,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False): """The agent config.""" model: Optional[str] - """The fully qualified name of the publisher model or endpoint to use for inference.""" + """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""" prompt_template: Optional[EvaluationRunPromptTemplateDict] """The prompt template used for inference.""" diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 6570f9578f..aa27b91dc0 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -228,6 +228,81 @@ def test_get_api_client_with_none_location( mock_agentplatform_client.assert_not_called() +class TestNormalizeInferenceModelName: + _FQ_PREFIX = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/" + + def test_short_gemini_name_expanded(self, mock_api_client_fixture): + result = _evals_common._normalize_inference_model_name( + "gemini-2.5-flash", mock_api_client_fixture + ) + assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash") + + def test_location_less_publisher_path_prepended(self, mock_api_client_fixture): + result = _evals_common._normalize_inference_model_name( + "publishers/google/models/gemini-2.5-flash", mock_api_client_fixture + ) + assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash") + + def test_third_party_publisher_path_prepended(self, mock_api_client_fixture): + result = _evals_common._normalize_inference_model_name( + "publishers/anthropic/models/claude-sonnet", mock_api_client_fixture + ) + assert result == (f"{self._FQ_PREFIX}publishers/anthropic/models/claude-sonnet") + + def test_endpoint_path_prepended(self, mock_api_client_fixture): + result = _evals_common._normalize_inference_model_name( + "endpoints/123", mock_api_client_fixture + ) + assert result == f"{self._FQ_PREFIX}endpoints/123" + + def test_models_path_canonicalized(self, mock_api_client_fixture): + result = _evals_common._normalize_inference_model_name( + "models/gemini-2.5-flash", mock_api_client_fixture + ) + assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash") + + def test_fully_qualified_name_unchanged(self, mock_api_client_fixture): + fq = f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash" + assert ( + _evals_common._normalize_inference_model_name(fq, mock_api_client_fixture) + == fq + ) + + def test_empty_model_unchanged(self, mock_api_client_fixture): + assert ( + _evals_common._normalize_inference_model_name("", mock_api_client_fixture) + == "" + ) + + def test_missing_project_or_location_raises(self, mock_api_client_fixture): + mock_api_client_fixture.location = None + with pytest.raises(ValueError, match="missing a project or location"): + _evals_common._normalize_inference_model_name( + "gemini-2.5-flash", mock_api_client_fixture + ) + + def test_unrecognized_model_raises(self, mock_api_client_fixture): + with pytest.raises(ValueError, match="Unrecognized model name"): + _evals_common._normalize_inference_model_name( + "not/a/valid/model", mock_api_client_fixture + ) + + def test_resolve_inference_configs_normalizes_model(self, mock_api_client_fixture): + inference_configs = { + "candidate-1": agentplatform_genai_types.EvaluationRunInferenceConfig( + model="gemini-2.5-flash" + ) + } + result = _evals_common._resolve_inference_configs( + mock_api_client_fixture, + common_types.EvaluationRunDataSource(), + inference_configs, + ) + assert result["candidate-1"].model == ( + f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash" + ) + + class TestTransformers: """Unit tests for transformers.""" diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index a33e923049..4936a60a96 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -513,9 +513,21 @@ def _resolve_inference_configs( else: config.agent_configs = parsed_agent_info.agents - # Resolve prompt template data if inference_configs: for inference_config in inference_configs.values(): + model_val = ( + inference_config.get("model") + if isinstance(inference_config, dict) + else inference_config.model + ) + if model_val: + normalized_model = _normalize_inference_model_name( + model_val, api_client + ) + if isinstance(inference_config, dict): + inference_config["model"] = normalized_model + else: + inference_config.model = normalized_model prompt_template_val = ( inference_config.get("prompt_template") if isinstance(inference_config, dict) @@ -912,6 +924,62 @@ def _is_gemini_model(model: str) -> bool: ) +def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str: + """Expands a model name to a fully-qualified resource name for inference. + + A short or location-less model name has no serving location for the + Evaluation Service to route on, so it is expanded using the client's + project and location. Already fully-qualified names pass through. Raises + ValueError if the client is missing a project or location, or if the model + name is not a recognized Vertex form. + """ + if not model: + return model + + if model.startswith("projects/"): + return model + + project = getattr(api_client, "project", None) + location = getattr(api_client, "location", None) + prefix = f"projects/{project}/locations/{location}/" + + def _require_project_location() -> None: + if not project or not location: + raise ValueError( + f"Cannot expand model name '{model}' to a fully-qualified" + " resource name because the client is missing a project or" + " location. Set project and location on the client, or pass a" + " fully-qualified" + " 'projects/{project}/locations/{location}/publishers/google/models/{model}'" + " resource name." + ) + + if ( + model.startswith("publishers/") + or model.startswith("endpoints/") + or model.startswith("tunedModels/") + ): + _require_project_location() + return f"{prefix}{model}" + + if model.startswith("models/"): + _require_project_location() + return f"{prefix}publishers/google/{model}" + + if "/" not in model and _is_gemini_model(model): + _require_project_location() + return f"{prefix}publishers/google/models/{model}" + + raise ValueError( + f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g." + " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint" + " resource name (e.g." + " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'" + " or" + " 'projects/{project}/locations/{location}/endpoints/{endpoint}')." + ) + + def _run_inference_internal( api_client: BaseApiClient, model: Union[Callable[[Any], Any], str], diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 33fff12b49..6f5a9a3e58 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -2659,6 +2659,11 @@ def create_evaluation_run( The key is the candidate name, and the value is the inference config. If provided, `agent_info` must be None. If omitted and `agent_info` is provided, this will be automatically constructed using `agent_info` and `user_simulator_config`. + The `model` field of an inference config accepts a short Gemini model + name (e.g. `gemini-2.5-flash`), which is automatically expanded to a + fully-qualified resource name using the client's project and location, + or an already fully-qualified publisher-model or endpoint resource + name. Example: {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")} labels: The labels to apply to the evaluation run. @@ -4448,6 +4453,11 @@ async def create_evaluation_run( The key is the candidate name, and the value is the inference config. If provided, `agent_info` must be None. If omitted and `agent_info` is provided, this will be automatically constructed using `agent_info` and `user_simulator_config`. + The `model` field of an inference config accepts a short Gemini model + name (e.g. `gemini-2.5-flash`), which is automatically expanded to a + fully-qualified resource name using the client's project and location, + or an already fully-qualified publisher-model or endpoint resource + name. Example: {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")} red_teaming_config: This field is experimental and may change in future diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 0579725846..f9ac6e1353 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2549,7 +2549,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel): ) model: Optional[str] = Field( default=None, - description="""The fully qualified name of the publisher model or endpoint to use for inference.""", + description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""", ) prompt_template: Optional[EvaluationRunPromptTemplate] = Field( default=None, description="""The prompt template used for inference.""" @@ -2574,7 +2574,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False): """The agent config.""" model: Optional[str] - """The fully qualified name of the publisher model or endpoint to use for inference.""" + """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""" prompt_template: Optional[EvaluationRunPromptTemplateDict] """The prompt template used for inference."""