diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index dd04f217ce..2008d1f053 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -91,6 +91,21 @@ _DEFAULT_CANDIDATE_NAME = _evals_constant.DEFAULT_CANDIDATE_NAME +def _local_timestamp() -> str: + """Returns the current local time as 'M/D/YYYY, H:MM:SS AM/PM'. + + Matches the Agent Platform UI's default experiment name timestamp format + (e.g. '6/1/2026, 1:12:29 PM'). + """ + now = datetime.datetime.now() + hour_12 = now.hour % 12 or 12 + meridiem = "AM" if now.hour < 12 else "PM" + return ( + f"{now.month}/{now.day}/{now.year}, " + f"{hour_12}:{now.minute:02d}:{now.second:02d} {meridiem}" + ) + + @contextlib.contextmanager def _temp_logger_level(logger_name: str, level: int) -> None: # type: ignore[misc] """Temporarily sets the level of a logger.""" diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index a3185e57df..4b600f7054 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -160,6 +160,13 @@ def _CreateEvaluationRunParameters_to_vertex( [item for item in getv(from_object, ["analysis_configs"])], ) + if getv(from_object, ["evaluation_experiment"]) is not None: + setv( + to_object, + ["evaluationExperiment"], + getv(from_object, ["evaluation_experiment"]), + ) + return to_object @@ -1415,6 +1422,7 @@ def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -1429,6 +1437,7 @@ def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -3216,6 +3225,7 @@ def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfoOrDict] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -3236,6 +3246,11 @@ def create_evaluation_run( metrics: The list of metrics to evaluate. name: The name of the evaluation run. display_name: The display name of the evaluation run. + evaluation_experiment: The resource name of an existing + EvaluationExperiment to group this run under. If omitted, a new + EvaluationExperiment is created automatically so the run is visible in + the Agent Platform UI. Pass an existing experiment name to group + multiple runs together. agent_info: The agent info to evaluate. Mutually exclusive with `inference_configs`. agent: The agent resource name in str type. Accepts either an Agent @@ -3394,10 +3409,22 @@ def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. return self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, @@ -4039,6 +4066,7 @@ async def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -4053,6 +4081,7 @@ async def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -5475,6 +5504,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -5495,6 +5525,11 @@ async def create_evaluation_run( metrics: The list of metrics to evaluate. name: The name of the evaluation run. display_name: The display name of the evaluation run. + evaluation_experiment: The resource name of an existing + EvaluationExperiment to group this run under. If omitted, a new + EvaluationExperiment is created automatically so the run is visible in + the Agent Platform UI. Pass an existing experiment name to group + multiple runs together. agent_info: The agent info to evaluate. Mutually exclusive with `inference_configs`. agent: The agent resource name in str type. Accepts either an Agent @@ -5653,11 +5688,23 @@ async def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. result = await self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index b52c21a36d..e2ff847bac 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -3017,6 +3017,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel): analysis_configs: Optional[list[AnalysisConfig]] = Field( default=None, description="""""" ) + evaluation_experiment: Optional[str] = Field( + default=None, + description="""The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) class _CreateEvaluationRunParametersDict(TypedDict, total=False): @@ -3046,6 +3052,11 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): analysis_configs: Optional[list[AnalysisConfigDict]] """""" + evaluation_experiment: Optional[str] + """The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + _CreateEvaluationRunParametersOrDict = Union[ _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict diff --git a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py index b89d794026..738a0b25b2 100644 --- a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py +++ b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py @@ -16,6 +16,7 @@ from tests.unit.agentplatform.genai.replays import pytest_helper from agentplatform import types +from agentplatform._genai import _evals_common from google.genai import types as genai_types import pandas as pd import pytest @@ -106,7 +107,7 @@ CANDIDATE_NAME = "candidate_1" MODEL_NAME = "projects/977012026409/locations/us-central1/publishers/google/models/gemini-2.5-flash" EVAL_SET_NAME = ( - "projects/977012026409/locations/us-central1/evaluationSets/6619939608513740800" + "projects/977012026409/locations/us-central1/evaluationSets/1936778737211146240" ) @@ -319,8 +320,13 @@ def test_create_eval_run_with_allow_cross_region_model(client): assert evaluation_run.error is None +@mock.patch.object( + _evals_common, "_local_timestamp", return_value="1/1/2026, 12:00:00 AM" +) @mock.patch("uuid.uuid4") -def test_create_eval_run_with_metric_resource_name(mock_uuid4, client): +def test_create_eval_run_with_metric_resource_name( + mock_uuid4, mock_local_timestamp, client +): """Tests create_evaluation_run with metric_resource_name.""" mock_uuid4.side_effect = [ uuid.UUID("d392c573-9e81-4a30-b984-8a6aa4656369"), @@ -718,7 +724,7 @@ def test_create_eval_run_with_interactions_data_source(mock_uuid4, client): """ mock_uuid4.return_value = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890") client._api_client._http_options.api_version = "v1beta1" - interaction_id = "ChA2YzllYzk0MjY1NjZjODM5EAgaATAqBG1haW4" + interaction_id = "ChBjYjdiNWIzYTMzMGQ2MjE1EAgaATAqBG1haW4" gemini_agent = ( "projects/model-evaluation-dev/locations/global/agents/test-agent-eval" ) @@ -914,7 +920,7 @@ def test_create_eval_run_with_gemini_agent(client): ) eval_set = ( "projects/model-evaluation-dev/locations/global/evaluationSets/" - "7392342128979869696" + "4227422097682464768" ) evaluation_run = client.evals.create_evaluation_run( name="test_gemini_agent", diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 9676271f73..67f4866538 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -10595,25 +10595,31 @@ def test_create_evaluation_run_config_default_is_none(self): def test_create_evaluation_run_passes_allow_cross_region_model(self): """Verifies allow_cross_region_model is sent inside evaluationConfig in the API request.""" evals_module = evals.Evals(api_client_=self.mock_api_client) - - evals_module.create_evaluation_run( - dataset=agentplatform_genai_types.EvaluationRunDataSource( - evaluation_set="projects/123/locations/us-central1/evaluationSets/789" - ), - metrics=[ - agentplatform_genai_types.EvaluationRunMetric( - metric="general_quality_v1", - metric_config=agentplatform_genai_types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name="general_quality_v1", - ) - ), - ) - ], - dest="gs://test-bucket/output", - config={"allow_cross_region_model": True}, + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ): + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + config={"allow_cross_region_model": True}, + ) + self.mock_api_client.request.assert_called_once() call_args = self.mock_api_client.request.call_args request_body = call_args[0][2] # Third positional arg is the request dict @@ -10629,25 +10635,33 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) return_value=self.mock_response ) async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) - - await async_evals_module.create_evaluation_run( - dataset=agentplatform_genai_types.EvaluationRunDataSource( - evaluation_set="projects/123/locations/us-central1/evaluationSets/789" - ), - metrics=[ - agentplatform_genai_types.EvaluationRunMetric( - metric="general_quality_v1", - metric_config=agentplatform_genai_types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name="general_quality_v1", - ) - ), - ) - ], - dest="gs://test-bucket/output", - config={"allow_cross_region_model": True}, + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ): + await async_evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + config={"allow_cross_region_model": True}, + ) + self.mock_api_client.async_request.assert_called_once() call_args = self.mock_api_client.async_request.call_args request_body = call_args[0][2] # Third positional arg is the request dict @@ -12315,3 +12329,179 @@ def test_delete_evaluation_experiment_uses_delete_and_full_name(self): call_args = self.mock_api_client.request.call_args assert call_args[0][0] == "delete" assert call_args[0][1] == self.experiment_name + + +class TestCreateEvaluationRunAutoExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationRuns/456", + "displayName": "test_run", + "state": "PENDING", + "evaluationExperiment": ( + "projects/123/locations/us-central1/evaluationExperiments/e1" + ), + } + ) + self.mock_api_client.request.return_value = self.mock_response + self.dataset = agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + self.metrics = [ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ] + + def test_auto_creates_experiment_when_not_provided(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + display_name="my_run", + ) + + mock_create_exp.assert_called_once() + assert mock_create_exp.call_args.kwargs["display_name"] == "my_run" + request_body = self.mock_api_client.request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/e1" + ) + + def test_auto_experiment_default_display_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + ) + + display_name = mock_create_exp.call_args.kwargs["display_name"] + assert display_name.startswith("SDK Experiment ") + + def test_uses_provided_experiment_without_creating(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + with mock.patch.object( + evals_module, "create_evaluation_experiment" + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + evaluation_experiment=( + "projects/123/locations/us-central1/evaluationExperiments/existing" + ), + ) + + mock_create_exp.assert_not_called() + request_body = self.mock_api_client.request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/existing" + ) + + +class TestAsyncCreateEvaluationRunAutoExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationRuns/456", + "displayName": "test_run", + "state": "PENDING", + "evaluationExperiment": ( + "projects/123/locations/us-central1/evaluationExperiments/e1" + ), + } + ) + self.mock_api_client.async_request = mock.AsyncMock( + return_value=self.mock_response + ) + self.dataset = agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + self.metrics = [ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ] + + @pytest.mark.asyncio + async def test_async_auto_creates_experiment_when_not_provided(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + display_name="my_run", + ) + + mock_create_exp.assert_awaited_once() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/e1" + ) + + @pytest.mark.asyncio + async def test_async_uses_provided_experiment_without_creating(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + evaluation_experiment=( + "projects/123/locations/us-central1/evaluationExperiments/existing" + ), + ) + + mock_create_exp.assert_not_awaited() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/existing" + ) diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 4d406b7b10..07618dcd78 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -65,6 +65,21 @@ AGENT_DATA = _evals_constant.AGENT_DATA +def _local_timestamp() -> str: + """Returns the current local time as 'M/D/YYYY, H:MM:SS AM/PM'. + + Matches the Agent Platform UI's default experiment name timestamp format + (e.g. '6/1/2026, 1:12:29 PM'). + """ + now = datetime.datetime.now() + hour_12 = now.hour % 12 or 12 + meridiem = "AM" if now.hour < 12 else "PM" + return ( + f"{now.month}/{now.day}/{now.year}, " + f"{hour_12}:{now.minute:02d}:{now.second:02d} {meridiem}" + ) + + @contextlib.contextmanager def _temp_logger_level(logger_name: str, level: int) -> None: # type: ignore[misc] """Temporarily sets the level of a logger.""" diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 625f64f5d6..3715571d96 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -137,6 +137,13 @@ def _CreateEvaluationRunParameters_to_vertex( [item for item in getv(from_object, ["analysis_configs"])], ) + if getv(from_object, ["evaluation_experiment"]) is not None: + setv( + to_object, + ["evaluationExperiment"], + getv(from_object, ["evaluation_experiment"]), + ) + return to_object @@ -1305,6 +1312,7 @@ def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -1319,6 +1327,7 @@ def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -2746,6 +2755,7 @@ def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfoOrDict] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -2892,10 +2902,22 @@ def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. return self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, @@ -3842,6 +3864,7 @@ async def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -3856,6 +3879,7 @@ async def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -4925,6 +4949,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -5071,11 +5096,23 @@ async def create_evaluation_run( self._api_client, resolved_dataset, inference_configs, parsed_agent_info ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) - resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name + # The backend rejects a caller-supplied `name` for runs that belong to an + # EvaluationExperiment and assigns the resource ID itself, so `name` is + # only used as a display_name fallback. result = await self._create_evaluation_run( - name=resolved_name, - display_name=display_name or resolved_name, + display_name=resolved_display_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index ac4947c9f2..17d95148d1 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2793,6 +2793,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel): analysis_configs: Optional[list[AnalysisConfig]] = Field( default=None, description="""""" ) + evaluation_experiment: Optional[str] = Field( + default=None, + description="""The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) class _CreateEvaluationRunParametersDict(TypedDict, total=False): @@ -2822,6 +2828,11 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): analysis_configs: Optional[list[AnalysisConfigDict]] """""" + evaluation_experiment: Optional[str] + """The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + _CreateEvaluationRunParametersOrDict = Union[ _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict