237 lines
10 KiB
Python
237 lines
10 KiB
Python
from deepeval.metrics import GEval
|
|
from deepeval.test_case import LLMTestCaseParams
|
|
|
|
from tests.conftest import claude
|
|
|
|
# Shared description of what this agent does, used as a base layer in all metric
|
|
# criteria so individual metrics don't need to re-explain the agent from scratch.
|
|
AGENT_CONTEXT = """
|
|
clan is a NixOS-based peer-to-peer computer management framework for self-hosting.
|
|
The agent is a clan deployment planner that turns natural language requests into
|
|
NixOS service configurations. It works by identifying the right service module from
|
|
a fixed catalog, then assigning that service's roles to specific machines or tags.
|
|
|
|
The agent's output is a structured configuration proposal — not shell commands,
|
|
step-by-step instructions, or general sysadmin advice. A valid response either:
|
|
- proposes a specific NixOS service that matches what the user asked for, or
|
|
- asks one focused clarifying question when the request is ambiguous or role/machine
|
|
assignments cannot be determined without more information.
|
|
""".strip()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Factory functions — each call returns a fresh GEval instance so that
|
|
# deepeval's assert_test never sees stale scores from a previous test case.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def helpfulness_metric() -> GEval:
|
|
"""Broad "was this useful?" baseline check."""
|
|
return GEval(
|
|
name="Helpfulness",
|
|
criteria=f"""
|
|
{AGENT_CONTEXT}
|
|
|
|
Given the above context, determine whether the 'actual output' is a helpful
|
|
response to the user's 'input'. Do NOT penalise the response for lacking shell commands or manual setup instructions — the agent's job is configuration
|
|
proposal, not operational guidance.
|
|
""",
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def configuration_correctness_metric() -> GEval:
|
|
"""Structural validity — no hallucinated names, valid schema shape."""
|
|
return GEval(
|
|
name="Configuration Correctness",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The 'context' contains the available machines, tags, and service schemas.",
|
|
"Check that every machine name in the 'actual output' exactly matches one "
|
|
"from the available machines in 'context'. Penalise any invented names.",
|
|
"Check that every tag name in the 'actual output' exactly matches one "
|
|
"from the available tags in 'context'. Penalise any invented names.",
|
|
"Check that the proposed configuration assigns at least one role to at least "
|
|
"one machine or tag — an empty configuration is invalid.",
|
|
"Verify the configuration follows the expected structure: "
|
|
'{"roles": {"role_name": {"machines": {...}, "tags": {...}}}}.',
|
|
"If the 'actual output' is a clarifying question instead of a configuration, "
|
|
"that is acceptable and should score neutrally (0.5), not be penalised.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
LLMTestCaseParams.CONTEXT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def service_match_metric() -> GEval:
|
|
"""Did the agent pick the right service?"""
|
|
return GEval(
|
|
name="Service Match Accuracy",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The 'expected output' contains the name of the correct service for this request.",
|
|
"Check whether the service proposed in 'actual output' matches the "
|
|
"'expected output'. An exact name match scores highest.",
|
|
"If the agent selected a different service that could still plausibly "
|
|
"serve the user's intent, give partial credit.",
|
|
"If the agent asked a clarifying question rather than selecting a service, "
|
|
"evaluate whether the question is reasonable given the ambiguity — "
|
|
"a well-motivated question should score neutrally, not be penalised.",
|
|
"If the agent selected a clearly wrong or unrelated service, score low.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
LLMTestCaseParams.EXPECTED_OUTPUT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def clarification_quality_metric() -> GEval:
|
|
"""Is the question focused, necessary, and singular?"""
|
|
return GEval(
|
|
name="Clarification Quality",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The agent should ask a clarifying question when the user's request "
|
|
"is genuinely ambiguous or when role/machine assignments cannot be "
|
|
"determined from the input alone.",
|
|
"Check that the question is focused and specific — it should target "
|
|
"exactly the missing piece of information (e.g., which machine, "
|
|
"which service, which role mapping).",
|
|
"Penalise vague questions like 'can you tell me more?' or questions "
|
|
"that ask for information already present in the input.",
|
|
"Penalise asking multiple questions at once — the agent should ask "
|
|
"ONE question per turn.",
|
|
"If the agent should have asked a question but instead guessed, "
|
|
"score low. If the agent asked a question when the answer was "
|
|
"already obvious from the input, also score low.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def scope_adherence_metric() -> GEval:
|
|
"""Does the agent stay within its defined responsibilities?"""
|
|
return GEval(
|
|
name="Scope Adherence",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The agent must ONLY propose NixOS service configurations from its "
|
|
"known catalog, or ask clarifying questions. It must NOT:",
|
|
" - Provide shell commands, manual setup instructions, or operational guidance.",
|
|
" - Offer general sysadmin advice unrelated to clan service configuration.",
|
|
" - Hallucinate services that are not in its catalog.",
|
|
" - Invent machine or tag names that were not provided.",
|
|
" - Engage with requests that are clearly outside the domain of "
|
|
" NixOS/clan service deployment (e.g., cooking recipes, trivia).",
|
|
"If the 'input' is an out-of-scope request, the agent should politely "
|
|
"redirect or decline. Score high if it does so, low if it engages.",
|
|
"If the 'input' is in-scope and the agent responds appropriately, score high.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
],
|
|
threshold=0.7,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def groundedness_metric() -> GEval:
|
|
"""Faithful to README docs, no fabricated features."""
|
|
return GEval(
|
|
name="Groundedness",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The 'retrieval context' contains the README documentation that was "
|
|
"provided to the agent for the selected service(s).",
|
|
"Check that any claims the agent makes about a service's capabilities, "
|
|
"roles, or constraints are supported by the retrieval context.",
|
|
"Penalise fabricated features, invented role names, or descriptions "
|
|
"that contradict the README documentation.",
|
|
"The agent is allowed to synthesize and rephrase — it does NOT need "
|
|
"to quote the README verbatim. But the substance must be faithful.",
|
|
"If the agent asks a clarifying question without making factual claims, "
|
|
"score neutrally — there is nothing to verify.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
LLMTestCaseParams.RETRIEVAL_CONTEXT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def conciseness_metric() -> GEval:
|
|
"""Terse, no preamble, no leaked system prompt."""
|
|
return GEval(
|
|
name="Conciseness",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"The agent should follow strict response discipline: either a tool "
|
|
"call / configuration proposal OR a clarifying question, but not both "
|
|
"mixed with lengthy explanation.",
|
|
"Penalise responses that repeat or paraphrase system prompt instructions.",
|
|
"Penalise unnecessary preamble, hedging, or filler before the actual "
|
|
"content (e.g., 'Sure! I'd be happy to help you with that.').",
|
|
"A good configuration proposal states what will be configured and on "
|
|
"which machines/tags, without over-explaining the service itself.",
|
|
"A good clarifying question is direct and under ~2 sentences.",
|
|
"Brevity is preferred — shorter responses that convey the same "
|
|
"information should score higher.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|
|
|
|
|
|
def task_completion_metric() -> GEval:
|
|
"""Actually produces a config for unambiguous requests."""
|
|
return GEval(
|
|
name="Task Completion",
|
|
evaluation_steps=[
|
|
f"{AGENT_CONTEXT}",
|
|
"For a clear, unambiguous 'input' that maps to a known service, "
|
|
"the agent MUST produce a concrete configuration proposal. "
|
|
"Failing to do so (e.g., asking unnecessary clarifying questions, "
|
|
"producing an empty or vague response) should score low.",
|
|
"The configuration proposal must include at least one role assigned "
|
|
"to at least one machine or tag.",
|
|
"If the 'input' is genuinely ambiguous, asking a clarifying question "
|
|
"counts as a valid outcome — score neutrally.",
|
|
"If the 'input' is out of scope, a polite refusal counts as "
|
|
"task completion — score neutrally.",
|
|
"Score highest when the agent completes the full pipeline: "
|
|
"identifies the service, assigns roles, and proposes configuration.",
|
|
],
|
|
evaluation_params=[
|
|
LLMTestCaseParams.INPUT,
|
|
LLMTestCaseParams.ACTUAL_OUTPUT,
|
|
],
|
|
threshold=0.5,
|
|
model=claude,
|
|
)
|