Evaluating AI Applications in .NET: Quality, Safety and CI

using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;

public sealed class MustMentionContext(IReadOnlyList<string> phrases)
    : EvaluationContext(ContextName, string.Join(", ", phrases))
{
    public const string ContextName = "Must Mention";
    public IReadOnlyList<string> Phrases { get; } = phrases;
}

public sealed class MustMentionEvaluator : IEvaluator
{
    public const string MetricName = "Required Facts";

    public IReadOnlyCollection<string> EvaluationMetricNames => [MetricName];

    public ValueTask<EvaluationResult> EvaluateAsync(
        IEnumerable<ChatMessage> messages,
        ChatResponse modelResponse,
        ChatConfiguration? chatConfiguration = null,
        IEnumerable<EvaluationContext>? additionalContext = null,
        CancellationToken cancellationToken = default)
    {
        IReadOnlyList<string> phrases =
            additionalContext?.OfType<MustMentionContext>().FirstOrDefault()?.Phrases ?? [];
        var missing = phrases
            .Where(p => !modelResponse.Text.Contains(p, StringComparison.OrdinalIgnoreCase))
            .ToList();

        var passed = missing.Count == 0;
        var reason = passed
            ? "All required facts present."
            : $"Missing: {string.Join(", ", missing)}";
        var metric = new BooleanMetric(MetricName, passed, reason)
        {
            Interpretation = passed
                ? new EvaluationMetricInterpretation(EvaluationRating.Good, reason: reason)
                : new EvaluationMetricInterpretation(EvaluationRating.Unacceptable, true, reason),
        };

        return ValueTask.FromResult(new EvaluationResult(metric));
    }
}

using Microsoft.Extensions.AI.Evaluation;
using Microsoft.Extensions.AI.Evaluation.Quality;
using Microsoft.Extensions.AI.Evaluation.Reporting;
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;

public static class EvalSetup
{
    public static readonly string AppModel =
        Environment.GetEnvironmentVariable("EVAL_APP_MODEL") ?? "gpt-5-mini";

    private static readonly string StorageRoot =
        Environment.GetEnvironmentVariable("EVAL_STORAGE_ROOT") ?? "eval-results";

    private static readonly string ExecutionName =
        Environment.GetEnvironmentVariable("GITHUB_RUN_ID")
        ?? $"local-{DateTime.UtcNow:yyyyMMddHHmm}";

    // Declared after the fields above so that static initialization order is correct.
    public static readonly ReportingConfiguration Reporting =
        DiskBasedReportingConfiguration.Create(
            storageRootPath: StorageRoot,
            evaluators:
            [
                new RelevanceEvaluator(),
                new CoherenceEvaluator(),
                new GroundednessEvaluator(),
                new MustMentionEvaluator(),
            ],
            chatConfiguration: new ChatConfiguration(JudgeClients.Create()),
            enableResponseCaching: true,
            cachingKeys: [AppModel, SupportPrompts.Version],
            executionName: ExecutionName,
            tags: [AppModel, $"prompt-{SupportPrompts.Version}"]);
}

using Microsoft.Extensions.AI.Evaluation;
using Microsoft.Extensions.AI.Evaluation.Quality;
using Microsoft.Extensions.AI.Evaluation.Reporting;
using Xunit;

public sealed class SupportAnswerEvaluations(SupportAppFixture app)
    : IClassFixture<SupportAppFixture>
{
    private static readonly IReadOnlyList<GoldenCase> Cases =
        GoldenDataset.Load("golden/support.json");

    public static TheoryData<string> CaseIds()
    {
        var data = new TheoryData<string>();
        foreach (var c in Cases) data.Add(c.Id);
        return data;
    }

    [Theory]
    [MemberData(nameof(CaseIds))]
    [Trait("Category", "Evaluation")]
    public async Task Answer_meets_the_quality_bar(string caseId)
    {
        var ct = TestContext.Current.CancellationToken;
        var testCase = Cases.Single(c => c.Id == caseId);

        await using ScenarioRun run = await EvalSetup.Reporting.CreateScenarioRunAsync(
            $"Support.{testCase.Id}", additionalTags: [testCase.Category], cancellationToken: ct);

        var (messages, response, chunks) = await app.AskAsync(testCase.Question, ct);

        EvaluationResult result = await run.EvaluateAsync(
            messages,
            response,
            additionalContext:
            [
                new GroundednessEvaluatorContext(string.Join("\n\n", chunks)),
                new MustMentionContext(testCase.MustMention),
            ],
            cancellationToken: ct);

        foreach (EvaluationMetric metric in result.Metrics.Values)
        {
            Assert.False(metric.Interpretation?.Failed ?? false, $"{metric.Name}: {metric.Reason}");
        }
    }
}