From ff0debbaf06a3e3ae0b98c90d3f052ae491d78ec Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Tue, 21 Jul 2026 11:34:42 -0700 Subject: [PATCH] feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity PiperOrigin-RevId: 951606606 --- agentplatform/_genai/evals.py | 24 +++- tests/unit/agentplatform/genai/test_evals.py | 125 ++++++++++++++++--- vertexai/_genai/evals.py | 19 ++- 3 files changed, 145 insertions(+), 23 deletions(-) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 50e3d868a2..4b600f7054 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -5504,6 +5504,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -5524,6 +5525,11 @@ async def create_evaluation_run( metrics: The list of metrics to evaluate. name: The name of the evaluation run. display_name: The display name of the evaluation run. + evaluation_experiment: The resource name of an existing + EvaluationExperiment to group this run under. If omitted, a new + EvaluationExperiment is created automatically so the run is visible in + the Agent Platform UI. Pass an existing experiment name to group + multiple runs together. agent_info: The agent info to evaluate. Mutually exclusive with `inference_configs`. agent: The agent resource name in str type. Accepts either an Agent @@ -5682,11 +5688,23 @@ async def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. result = await self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 2a9ca98507..67f4866538 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -10635,25 +10635,33 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) return_value=self.mock_response ) async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) - - await async_evals_module.create_evaluation_run( - dataset=agentplatform_genai_types.EvaluationRunDataSource( - evaluation_set="projects/123/locations/us-central1/evaluationSets/789" - ), - metrics=[ - agentplatform_genai_types.EvaluationRunMetric( - metric="general_quality_v1", - metric_config=agentplatform_genai_types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name="general_quality_v1", - ) - ), - ) - ], - dest="gs://test-bucket/output", - config={"allow_cross_region_model": True}, + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ): + await async_evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + config={"allow_cross_region_model": True}, + ) + self.mock_api_client.async_request.assert_called_once() call_args = self.mock_api_client.async_request.call_args request_body = call_args[0][2] # Third positional arg is the request dict @@ -12414,3 +12422,86 @@ def test_uses_provided_experiment_without_creating(self): request_body.get("evaluationExperiment") == "projects/123/locations/us-central1/evaluationExperiments/existing" ) + + +class TestAsyncCreateEvaluationRunAutoExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationRuns/456", + "displayName": "test_run", + "state": "PENDING", + "evaluationExperiment": ( + "projects/123/locations/us-central1/evaluationExperiments/e1" + ), + } + ) + self.mock_api_client.async_request = mock.AsyncMock( + return_value=self.mock_response + ) + self.dataset = agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + self.metrics = [ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ] + + @pytest.mark.asyncio + async def test_async_auto_creates_experiment_when_not_provided(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + display_name="my_run", + ) + + mock_create_exp.assert_awaited_once() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/e1" + ) + + @pytest.mark.asyncio + async def test_async_uses_provided_experiment_without_creating(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + evaluation_experiment=( + "projects/123/locations/us-central1/evaluationExperiments/existing" + ), + ) + + mock_create_exp.assert_not_awaited() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/existing" + ) diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 963b4fb121..3715571d96 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -4949,6 +4949,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -5095,11 +5096,23 @@ async def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. result = await self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs,