Implementation:Microsoft Autogen Studio Eval Judges

Sources	python/packages/autogen-studio/autogenstudio/eval/judges.py
Domains	Evaluation, LLM, Agent_Systems, Judging
Last Updated	2026-02-11

Overview

Description

The Studio Eval Judges module provides an extensible framework for evaluating agent task results using large language models as judges. This implementation includes abstract base classes and a concrete LLM-based judge that can assess results across multiple evaluation dimensions.

The module defines:

• BaseEvalJudge - Abstract base class defining the judge interface with serialization support
• BaseEvalJudgeConfig - Configuration model for base judge parameters
• LLMEvalJudge - Concrete implementation that uses an LLM to evaluate results against criteria
• LLMEvalJudgeConfig - Configuration model including model client settings

The LLM judge can evaluate results across multiple dimensions simultaneously using parallel evaluation, producing structured scores with reasoning for each dimension. It leverages AutoGen's component system for serialization and deserialization.

Usage

Judges are used in evaluation pipelines to assess the quality of agent responses. The LLMEvalJudge takes evaluation tasks and results, along with criteria, and produces EvalScore objects containing dimension-specific scores and overall scores. The judge system is designed to be extensible, allowing developers to create custom judges by extending BaseEvalJudge.

Code Reference

Source Location

python/packages/autogen-studio/autogenstudio/eval/judges.py

Signature

class BaseEvalJudgeConfig(BaseModel):
    """Base configuration for evaluation judges."""
    name: str = "Base Judge"
    description: str = ""
    metadata: Dict[str, Any] = {}

class BaseEvalJudge(ABC, ComponentBase[BaseEvalJudgeConfig]):
    """Abstract base class for evaluation judges."""
    component_type = "eval_judge"

    def __init__(
        self,
        name: str = "Base Judge",
        description: str = "",
        metadata: Optional[Dict[str, Any]] = None
    )

    @abstractmethod
    async def judge(
        self,
        task: EvalTask,
        result: EvalRunResult,
        criteria: List[EvalJudgeCriteria],
        cancellation_token: Optional[CancellationToken] = None,
    ) -> EvalScore

    def _to_config(self) -> BaseEvalJudgeConfig

class LLMEvalJudgeConfig(BaseEvalJudgeConfig):
    """Configuration for LLMEvalJudge."""
    model_client: Any  # ComponentModel

class LLMEvalJudge(BaseEvalJudge, Component[LLMEvalJudgeConfig]):
    """Judge that uses an LLM to evaluate results."""
    component_config_schema = LLMEvalJudgeConfig
    component_type = "eval_judge"
    component_provider_override = "autogenstudio.eval.judges.LLMEvalJudge"

    def __init__(
        self,
        model_client: ChatCompletionClient,
        name: str = "LLM Judge",
        description: str = "Evaluates results using an LLM",
        metadata: Optional[Dict[str, Any]] = None,
    )

    async def judge(
        self,
        task: EvalTask,
        result: EvalRunResult,
        criteria: List[EvalJudgeCriteria],
        cancellation_token: Optional[CancellationToken] = None,
    ) -> EvalScore

    async def _judge_dimension(
        self,
        task: EvalTask,
        result: EvalRunResult,
        criterion: EvalJudgeCriteria,
        cancellation_token: Optional[CancellationToken] = None,
    ) -> EvalDimensionScore

    def _format_task(self, task: EvalTask) -> str

    def _parse_judgment(
        self,
        judgment_text: str,
        max_value: float
    ) -> Tuple[str, Optional[float]]

    def _to_config(self) -> LLMEvalJudgeConfig

    @classmethod
    def _from_config(cls, config: LLMEvalJudgeConfig) -> Self

Import

from autogenstudio.eval.judges import (
    BaseEvalJudge,
    BaseEvalJudgeConfig,
    LLMEvalJudge,
    LLMEvalJudgeConfig
)

I/O Contract

Inputs

Parameter	Type	Description
task	EvalTask	The evaluation task containing input, description, and metadata
result	EvalRunResult	The result produced by running the evaluation task
criteria	List[EvalJudgeCriteria]	List of evaluation criteria, one per dimension to be judged
cancellation_token	Optional[CancellationToken]	Token to cancel the judging operation
model_client (LLMEvalJudge)	ChatCompletionClient	LLM client used to perform the evaluation

Outputs

Return Type	Description
EvalScore	Composite score containing overall_score (average of dimension scores), dimension_scores (list of EvalDimensionScore objects), and metadata
EvalDimensionScore (_judge_dimension)	Individual dimension score with dimension name, score value, reasoning text, and min/max bounds

Usage Examples

Creating and Using an LLM Judge

from autogenstudio.eval.judges import LLMEvalJudge
from autogen_ext.models.openai import OpenAIChatCompletionClient
from autogenstudio.datamodel.eval import EvalTask, EvalRunResult, EvalJudgeCriteria

# Create model client
model_client = OpenAIChatCompletionClient(
    model="gpt-4",
    api_key="your-api-key"
)

# Create LLM judge
llm_judge = LLMEvalJudge(
    model_client=model_client,
    name="GPT-4 Judge",
    description="Evaluates responses using GPT-4"
)

# Define evaluation criteria
criteria = [
    EvalJudgeCriteria(
        dimension="relevance",
        prompt="Evaluate how relevant the response is to the query.",
        min_value=0,
        max_value=10
    ),
    EvalJudgeCriteria(
        dimension="accuracy",
        prompt="Evaluate the factual accuracy of the response.",
        min_value=0,
        max_value=10
    ),
    EvalJudgeCriteria(
        dimension="completeness",
        prompt="Evaluate whether the response fully addresses the question.",
        min_value=0,
        max_value=10
    )
]

# Create task and result
task = EvalTask(
    name="Capital Query",
    description="Geography question",
    input="What is the capital of France?"
)

result = EvalRunResult(
    status=True,
    result=task_result  # TaskResult from runner
)

# Run evaluation
score = await llm_judge.judge(task, result, criteria)

print(f"Overall Score: {score.overall_score}")
for dim_score in score.dimension_scores:
    print(f"{dim_score.dimension}: {dim_score.score} - {dim_score.reason}")

Serializing and Deserializing a Judge

from autogenstudio.eval.judges import LLMEvalJudge
from autogen_ext.models.openai import OpenAIChatCompletionClient

# Create judge
model_client = OpenAIChatCompletionClient(
    model="gpt-4o-mini",
    api_key="your-api-key"
)

llm_judge = LLMEvalJudge(
    model_client=model_client,
    name="Mini Judge",
    description="Fast evaluation with GPT-4o-mini"
)

# Serialize to ComponentModel
judge_config = llm_judge.dump_component()
print(f"Serialized judge: {judge_config}")

# Save to JSON
import json
with open("judge_config.json", "w") as f:
    json.dump(judge_config.model_dump(), f, indent=2)

# Deserialize back to LLMEvalJudge
deserialized_judge = LLMEvalJudge.load_component(judge_config)

# Use deserialized judge
score = await deserialized_judge.judge(task, result, criteria)

Creating a Custom Judge

from autogenstudio.eval.judges import BaseEvalJudge, BaseEvalJudgeConfig
from autogenstudio.datamodel.eval import (
    EvalTask, EvalRunResult, EvalJudgeCriteria,
    EvalScore, EvalDimensionScore
)
from typing import List, Optional
from autogen_core import CancellationToken

class RuleBasedJudge(BaseEvalJudge):
    """Custom judge using rule-based evaluation."""

    def __init__(self, name: str = "Rule Judge", description: str = ""):
        super().__init__(name, description)

    async def judge(
        self,
        task: EvalTask,
        result: EvalRunResult,
        criteria: List[EvalJudgeCriteria],
        cancellation_token: Optional[CancellationToken] = None,
    ) -> EvalScore:
        """Implement custom judging logic."""
        dimension_scores = []

        for criterion in criteria:
            # Implement rule-based scoring logic
            score_value = self._calculate_rule_score(task, result, criterion)

            dim_score = EvalDimensionScore(
                dimension=criterion.dimension,
                score=score_value,
                reason=f"Rule-based score for {criterion.dimension}",
                max_value=criterion.max_value,
                min_value=criterion.min_value
            )
            dimension_scores.append(dim_score)

        # Calculate overall score
        overall = sum(ds.score for ds in dimension_scores) / len(dimension_scores)

        return EvalScore(
            overall_score=overall,
            dimension_scores=dimension_scores
        )

    def _calculate_rule_score(
        self,
        task: EvalTask,
        result: EvalRunResult,
        criterion: EvalJudgeCriteria
    ) -> float:
        """Custom scoring logic."""
        # Implement your rule-based scoring here
        if not result.status:
            return 0.0

        # Example: length-based scoring
        if result.result and result.result.messages:
            message_length = len(str(result.result.messages[-1].content))
            return min(message_length / 100, criterion.max_value)

        return 0.0

# Use custom judge
custom_judge = RuleBasedJudge(
    name="Length-Based Judge",
    description="Scores based on response length"
)

score = await custom_judge.judge(task, result, criteria)

Parallel Evaluation Across Multiple Dimensions

from autogenstudio.eval.judges import LLMEvalJudge
from autogenstudio.datamodel.eval import EvalJudgeCriteria

# Define comprehensive criteria
criteria = [
    EvalJudgeCriteria(
        dimension="relevance",
        prompt="Is the response relevant to the user's query?"
    ),
    EvalJudgeCriteria(
        dimension="accuracy",
        prompt="Is the response factually accurate?"
    ),
    EvalJudgeCriteria(
        dimension="clarity",
        prompt="Is the response clear and easy to understand?"
    ),
    EvalJudgeCriteria(
        dimension="completeness",
        prompt="Does the response fully address the query?"
    ),
    EvalJudgeCriteria(
        dimension="helpfulness",
        prompt="How helpful is the response to the user?"
    )
]

# The judge will evaluate all dimensions in parallel
score = await llm_judge.judge(task, result, criteria)

# Access individual dimension scores
for dim_score in score.dimension_scores:
    print(f"\n{dim_score.dimension.upper()}")
    print(f"Score: {dim_score.score}/{dim_score.max_value}")
    print(f"Reason: {dim_score.reason}")

print(f"\nOverall Score: {score.overall_score:.2f}/10.0")

Handling Evaluation Errors

from autogenstudio.eval.judges import LLMEvalJudge
import asyncio

try:
    # Run evaluation with timeout
    score = await asyncio.wait_for(
        llm_judge.judge(task, result, criteria),
        timeout=30.0  # 30 second timeout
    )

    # Check for failed dimensions
    for dim_score in score.dimension_scores:
        if dim_score.score == 0.0 and "Failed to parse" in dim_score.reason:
            print(f"Warning: Failed to evaluate {dim_score.dimension}")

except asyncio.TimeoutError:
    print("Evaluation timed out after 30 seconds")
except Exception as e:
    print(f"Evaluation failed: {e}")

Related Pages

• Studio Eval Datamodel - Data models used by judges
• Studio Eval Runners - Runners that produce results for judges
• Evaluation Domain - All evaluation-related implementations
• LLM Domain - Language model integrations
• Judging Domain - Quality assessment implementations