Add LLMRecommender for LLM-based experimental design

CHANGELOG.md

@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `df_apply_permutation_augmentation` has a different interface and now expects
   permutation groups instead of column groups
 - `ParameterSelectorProtocol.__call__` now declares its input as positional-only
+- `SequentialMetaRecommender.recommenders` is now a `tuple` instead of a `list`, making
+  the attribute immutable. Serialization is unaffected
 
 ### Fixed
 - `DiscretePermutationInvarianceConstraint` no longer erroneously removes points where
@@ -48,6 +50,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `DiscreteLinearConstraint` for (optionally weighted) sum constraints on discrete
   parameters, supporting `coefficients` and mirroring `ContinuousLinearConstraint`'s
   `operator`/`rhs`/`coefficients` interface (replaces `DiscreteSumConstraint`)
+- `LLMRecommender` for LLM-based experimental design suggestions via LiteLLM,
+  available through the optional `llm` dependency group
+- `LLMTwoPhaseRecommender` and `LLMAlternatingRecommender` convenience recommenders
+  built on top of `LLMRecommender`: the former warm-starts a campaign with the LLM and
+  then switches to another recommender, the latter alternates between the two
 
 ### Changed
 - `BOTORCH` GP preset now includes `BetaPrior(2.5, 1.5)` for the task covariance

baybe/_optional/info.py

@@ -25,7 +25,9 @@ def exclude_sys_path(path: str, /):  # noqa: DOC402
 # Individual packages
 with exclude_sys_path(os.getcwd()):
     FLAKE8_INSTALLED = find_spec("flake8") is not None
+    JINJA2_INSTALLED = find_spec("jinja2") is not None
     LIME_INSTALLED = find_spec("lime") is not None
+    LITELLM_INSTALLED = find_spec("litellm") is not None
     ONNX_INSTALLED = find_spec("onnxruntime") is not None
     NGBOOST_INSTALLED = find_spec("ngboost") is not None
     POLARS_INSTALLED = find_spec("polars") is not None
@@ -49,6 +51,7 @@ def exclude_sys_path(path: str, /):  # noqa: DOC402
 # Information on whether all required packages for certain functionality are available
 CHEM_INSTALLED = SKFP_INSTALLED
 INSIGHTS_INSTALLED = SHAP_INSTALLED and LIME_INSTALLED
+LLM_INSTALLED = JINJA2_INSTALLED and LITELLM_INSTALLED
 LINT_INSTALLED = all(
     (
         FLAKE8_INSTALLED,

baybe/_optional/llm.py

@@ -0,0 +1,20 @@
+"""Optional LLM imports."""
+
+from baybe.exceptions import OptionalImportError
+
+try:
+    from jinja2 import StrictUndefined, Template
+except ModuleNotFoundError as ex:
+    raise OptionalImportError(name="jinja2", group="llm") from ex
+
+try:
+    from litellm import AuthenticationError, completion
+except ModuleNotFoundError as ex:
+    raise OptionalImportError(name="litellm", group="llm") from ex
+
+__all__ = [
+    "AuthenticationError",
+    "StrictUndefined",
+    "Template",
+    "completion",
+]

baybe/exceptions.py

@@ -1,6 +1,7 @@
 """Custom exceptions and warnings."""
 
 import gc
+from collections.abc import Collection
 from typing import Any
 
 import pandas as pd
@@ -15,6 +16,10 @@ class InputDataTypeWarning(UserWarning):
     """An input has unexpected data type."""
 
 
+class LLMResponseWarning(UserWarning):
+    """A language model response did not fully meet the request."""
+
+
 class UnusedObjectWarning(UserWarning):
     """
     A method or function was called with undesired arguments which indicates an
@@ -187,5 +192,184 @@ class UnsupportedEarlyFilteringError(Exception):
     """A constraint does not support early filtering with the given parameters."""
 
 
+#### LLM exclusive exceptions ####
+
+
+class LLMError(Exception):
+    """A language model interaction failed."""
+
+
+class LLMCallError(LLMError):
+    """The call to the language model failed before producing a response.
+
+    Raised for failures such as network errors, rate limiting, timeouts, or an
+    unknown model identifier, where no response is returned to process.
+    """
+
+
+class LLMAuthenticationError(LLMCallError):
+    """Authentication with the language model provider failed."""
+
+
+class LLMResponseError(LLMError):
+    """An error occurred while processing a language model response."""
+
+    @property
+    def recovery_instruction(self) -> str:
+        """Guidance for the language model on how to correct its response."""
+        return (
+            "Your previous response could not be used. Please provide a new "
+            "recommendation that follows the required format and respects the search "
+            "space."
+        )
+
+
+class LLMMalformedResponseError(LLMResponseError):
+    """A language model response could not be parsed into the expected structure."""
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            "Your previous response was not valid JSON in the required format. Return "
+            "a JSON array of objects, each with an 'explanation' string and a "
+            "'parameters' object mapping parameter names to values, and nothing else."
+        )
+
+
+class LLMUnknownParameterError(LLMResponseError):
+    """A language model response referenced parameters outside the search space."""
+
+    def __init__(
+        self,
+        *args: Any,
+        unknown_names: Collection[str],
+        valid_names: Collection[str],
+    ):
+        super().__init__(*args)
+        self.unknown_names = unknown_names
+        self.valid_names = valid_names
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response used unknown parameter names "
+            f"{sorted(self.unknown_names)}. Use only the following parameters: "
+            f"{sorted(self.valid_names)}."
+        )
+
+
+class LLMMissingParameterError(LLMResponseError):
+    """A language model response omitted required search space parameters."""
+
+    def __init__(self, *args: Any, parameters: Collection[str]):
+        super().__init__(*args)
+        self.parameters = parameters
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response did not specify values for the required "
+            f"parameters {sorted(self.parameters)}. Every suggestion must provide a "
+            f"value for all parameters."
+        )
+
+
+class LLMNonNumericParameterError(LLMResponseError):
+    """A language model response gave non-numeric values for a numerical parameter."""
+
+    def __init__(self, *args: Any, detail: str):
+        super().__init__(*args)
+        self.detail = detail
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response provided non-numeric values for a numerical "
+            f"parameter: {self.detail} Provide numeric values for all numerical "
+            f"parameters."
+        )
+
+
+class LLMInvalidParameterValueError(LLMResponseError):
+    """A language model response contained invalid parameter values."""
+
+    def __init__(self, *args: Any, detail: str):
+        super().__init__(*args)
+        self.detail = detail
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response contained invalid parameter values: {self.detail} "
+            f"Provide values that lie within the allowed range (for numerical "
+            f"parameters) or match an allowed choice exactly (for categorical "
+            f"parameters)."
+        )
+
+
+class LLMConstraintViolationError(LLMResponseError):
+    """A language model response violated a search space constraint."""
+
+    def __init__(
+        self,
+        *args: Any,
+        constraint_name: str,
+        parameters: Collection[str],
+    ):
+        super().__init__(*args)
+        self.constraint_name = constraint_name
+        self.parameters = parameters
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response violated the '{self.constraint_name}' constraint "
+            f"on parameters {list(self.parameters)}. Ensure all suggestions satisfy "
+            f"this constraint."
+        )
+
+
+class LLMIneligiblePointsError(LLMResponseError):
+    """A language model response proposed points outside the eligible candidate set."""
+
+    def __init__(self, *args: Any, n_ineligible: int):
+        super().__init__(*args)
+        self.n_ineligible = n_ineligible
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"{self.n_ineligible} of your proposed configurations are not eligible "
+            f"candidates (they may already have been measured or recommended, or they "
+            f"do not exist in the search space). Propose different points from the "
+            f"available search space."
+        )
+
+
+class LLMBatchSizeError(LLMResponseError):
+    """A language model response contained the wrong number of recommendations."""
+
+    def __init__(self, *args: Any, requested: int, received: int):
+        super().__init__(*args)
+        self.requested = requested
+        self.received = received
+
+    @property
+    @override
+    def recovery_instruction(self) -> str:
+        return (
+            f"Your previous response provided {self.received} valid recommendation(s), "
+            f"but exactly {self.requested} are required. Provide {self.requested} "
+            f"distinct, valid recommendations."
+        )
+
+
 # Collect leftover original slotted classes processed by `attrs.define`
 gc.collect()

baybe/recommenders/__init__.py

@@ -1,12 +1,17 @@
 """BayBE recommenders."""
 
+from baybe.recommenders.meta.llm import (
+    LLMAlternatingRecommender,
+    LLMTwoPhaseRecommender,
+)
 from baybe.recommenders.meta.sequential import (
     SequentialMetaRecommender,
     StreamingSequentialMetaRecommender,
     TwoPhaseMetaRecommender,
 )
 from baybe.recommenders.naive import NaiveHybridSpaceRecommender
 from baybe.recommenders.pure.bayesian.botorch import BotorchRecommender
+from baybe.recommenders.pure.llm import LLMRecommender
 from baybe.recommenders.pure.nonpredictive.clustering import (
     GaussianMixtureClusteringRecommender,
     KMeansClusteringRecommender,
@@ -22,6 +27,9 @@
     "FPSRecommender",
     "GaussianMixtureClusteringRecommender",
     "KMeansClusteringRecommender",
+    "LLMAlternatingRecommender",
+    "LLMRecommender",
+    "LLMTwoPhaseRecommender",
     "PAMClusteringRecommender",
     "NaiveHybridSpaceRecommender",
     "RandomRecommender",

baybe/recommenders/meta/__init__.py

@@ -5,13 +5,19 @@
 query.
 """
 
+from baybe.recommenders.meta.llm import (
+    LLMAlternatingRecommender,
+    LLMTwoPhaseRecommender,
+)
 from baybe.recommenders.meta.sequential import (
     SequentialMetaRecommender,
     StreamingSequentialMetaRecommender,
     TwoPhaseMetaRecommender,
 )
 
 __all__ = [
+    "LLMAlternatingRecommender",
+    "LLMTwoPhaseRecommender",
     "TwoPhaseMetaRecommender",
     "SequentialMetaRecommender",
     "StreamingSequentialMetaRecommender",