RAG Is Easy to Build, Hard to Trust: Evaluation to the Rescue

ollama pull llama3.2:latest
ollama pull nomic-embed-text

using MCP_Aspire.Domain;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
 
var builder = DistributedApplication.CreateBuilder(args);
var ragMode = builder.Configuration.GetValue("RAGApi:Mode", RagMode.Graph);
if (!Enum.IsDefined(ragMode))
{
    throw new InvalidOperationException("RAGApi:Mode must be Graph or Vector.");
}
var captureMessageContent = builder.Configuration["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"]
    ?? (builder.Environment.IsDevelopment() ? "true" : "false");
 
Console.WriteLine($"Starting {ragMode} RAG.");
 
var chat = builder.AddConnectionString("chat");
 
// Arize Phoenix - Open-source AI observability & evaluation
var phoenix = builder.AddContainer("phoenix", "arizephoenix/phoenix", "latest")
    .WithContainerName("phoenix")
    .WithHttpEndpoint(port: builder.Configuration.GetValue("Phoenix:Port", 6006), targetPort: 6006)
    .WithUrlForEndpoint("http", url => url.DisplayText = "Phoenix")
    .WithEnvironment("PHOENIX_WORKING_DIR", "/mnt/data")
    .WithVolume("mcp-aspire-phoenix-data", "/mnt/data")
    .WithHttpHealthCheck("/healthz");
 
// MLflow - debug, evaluate, and monitor your LLM applications, Agents, and Models.
var experimentId = builder.Configuration["MLflow:ExperimentId"] ?? "0";
var mlflow = builder.AddContainer("mlflow", "ghcr.io/mlflow/mlflow", "latest")
    .WithContainerName("mlflow")
    .WithHttpEndpoint(port: builder.Configuration.GetValue("MLflow:Port", 5001), targetPort: 5000)
    .WithUrlForEndpoint("http", url => url.DisplayText = "MLflow")
    .WithVolume(builder.Configuration["MLflow:VolumeName"] ?? "mcp-aspire-mlflow-data", "/mlflow")
    .WithArgs("mlflow", "server", "--host", "0.0.0.0", "--port", "5000",
        "--backend-store-uri", "sqlite:////mlflow/mlflow.db",
        "--default-artifact-root", "/mlflow/artifacts")
    .WithHttpHealthCheck("/health");
 
// RAG API - the main application that provides retrieval-augmented generation capabilities.
var ragapi = builder.AddProject<Projects.MCP_Aspire_RAGApi>("ragapi")
    .WithEnvironment("RAGApi__Mode", ragMode.ToString())
    .WithHttpHealthCheck("/health")
    .WithEnvironment("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", captureMessageContent)
    .WithEnvironment("Phoenix__Endpoint", phoenix.GetEndpoint("http"))
    .WithEnvironment("Phoenix__ProjectName", builder.Configuration["Phoenix:ProjectName"] ?? "ragapi")
    .WaitFor(phoenix)
    .WithEnvironment("MLflow__TrackingUri", mlflow.GetEndpoint("http"))
    .WithEnvironment("MLflow__ExperimentId", experimentId)
    .WaitFor(mlflow)
    .WithReference(chat)
    .WithUrlForEndpoint("http", url =>
    {
        url.Url = "/Scalar";
        url.DisplayText = "RAG API";
    })
    .WithUrlForEndpoint("https", url =>
    {
        url.Url = "/Scalar";
        url.DisplayText = "RAG API";
    });
 
if (ragMode == RagMode.Graph)
{
    var isGraphToolsEnabled = builder.Configuration.GetValue("RAGApi:GraphEnableTools", true);
    ragapi.WithEnvironment("RAGApi__GraphEnableTools", isGraphToolsEnabled.ToString());
}
else
{
    var embedding = builder.AddConnectionString("embedding");
    ragapi.WithReference(embedding);
}
 
// Evaluation CLI - a command-line tool for evaluating the RAG API and generating reports.
if (builder.ExecutionContext.IsRunMode)
{
    var reportDirectory = Path.GetFullPath(Path.Combine(builder.AppHostDirectory, "..", "artifacts", "rag-evaluation"));
 
    builder.AddProject<Projects.MCP_Aspire_RAGApi_Evaluation_Cli>("evaluation")
        .WithExplicitStart()
        .WaitFor(ragapi)
        .WithArgs(context =>
        {
            // Recompute on every launch so dashboard reruns preserve previous reports.
            var reportName = $"{DateTime.UtcNow:yyyyMMdd-HHmmss-fffffff}";
            context.Args.Add("--base-url");
            context.Args.Add(ragapi.GetEndpoint("http"));
            context.Args.Add("--exit-code-policy");
            context.Args.Add("completion");
            context.Args.Add("--output");
            context.Args.Add(Path.Combine(reportDirectory, $"{reportName}.html"));
            context.Args.Add("--json-output");
            context.Args.Add(Path.Combine(reportDirectory, $"{reportName}.json"));
        });
}
 
builder.Build().Run();

{
  "RAGApi": {
    "Mode": "Vector" // Change to Graph, to run in Graph mode
  },
  "ConnectionStrings": {
    "chat": "Endpoint=http://localhost:11434;Model=llama3.2:latest",
    "embedding": "Endpoint=http://localhost:11434;Model=nomic-embed-text"
  },
  "MLflow": {
    "ExperimentId": "0"
  },
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft.AspNetCore": "Warning",
      "Aspire.Hosting.Dcp": "Warning"
    }
  }
}

namespace MCP_Aspire.Domain.Graph;
 
public sealed record AnswerRunMetadata(
    string Model,
    string PromptFingerprint,
    int MaxAnswerTokens,
    int ContextTokens,
    double Temperature,
    long Seed,
    string Variant)
{
    public const string HeaderName = "X-Rag-Answer-Run";
    public string? ContextPolicy { get; init; }
    public string? DiscoveryPromptFingerprint { get; init; }
    public string? RepairPromptFingerprint { get; init; }
    public int ModelCalls { get; init; }
    public string? FinishReason { get; init; }
    public long? InputTokens { get; init; }
    public long? OutputTokens { get; init; }
}
 
public sealed record GraphAnswerResult(string Answer, AnswerRunMetadata Run);

namespace MCP_Aspire.Domain.Graph;
 
public static class GraphAnswerContract
{
    public const string NoEvidenceAnswer = "I could not find supporting information in the knowledge graph to answer that question.";
}

namespace MCP_Aspire.Domain.Graph;
 
public sealed record GraphNode(
    string Id,
    string Type,
    string Label,
    IReadOnlyList<string> Aliases,
    IReadOnlyList<string> EvidenceIds);
 
public sealed record GraphEdge(string From, string Type, string To, IReadOnlyList<string> EvidenceIds);
 
public sealed record GraphEvidence(string Id, string SourceName, string Text);
 
public sealed record GraphSearchResult(
    IReadOnlyList<GraphNode> Nodes,
    IReadOnlyList<GraphEdge> Edges,
    IReadOnlyList<GraphEvidence> Evidence,
    bool Truncated);

namespace MCP_Aspire.Domain;
 
public enum RagMode
{
    Graph,
    Vector
}

namespace MCP_Aspire.Domain;
 
public sealed record RagModeResponse(string Mode);

namespace MCP_Aspire.Domain;
 
public record SearchRequest
{
    public required string Query { get; init; }
}

dotnet add package Microsoft.Extensions.AI --version 10.8.0
dotnet add package Prompty.Core --version 2.0.0

using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using MCP_Aspire.Domain.Graph;
using Microsoft.Extensions.AI;
using Prompty.Core;
 
namespace MCP_Aspire.Prompts;
 
public sealed class GraphAnswerPromptRenderer
{
    private readonly Agent prompt;
    private readonly Agent discoveryPrompt;
    private readonly Agent repairPrompt;
 
    public GraphAnswerPromptRenderer()
        : this(Path.Combine(AppContext.BaseDirectory, "Graph", "graph-answer.prompty"))
    {
    }
 
    public GraphAnswerPromptRenderer(string templatePath)
        : this(templatePath, null, null)
    {
    }
 
    public GraphAnswerPromptRenderer(string templatePath, string? discoveryTemplatePath = null,
        string? repairTemplatePath = null)
    {
        NonceSafePromptyChatParser.Register();
        string template = File.ReadAllText(templatePath);
        prompt = PromptyLoader.Load(templatePath);
        string contract = (template + "\n" + GraphAnswerContract.NoEvidenceAnswer)
            .ReplaceLineEndings("\n");
        Fingerprint = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(contract)));
        discoveryTemplatePath ??= Path.Combine(AppContext.BaseDirectory, "Graph", "graph-discovery.prompty");
        repairTemplatePath ??= Path.Combine(AppContext.BaseDirectory, "Graph", "graph-citation-repair.prompty");
        discoveryPrompt = PromptyLoader.Load(discoveryTemplatePath);
        repairPrompt = PromptyLoader.Load(repairTemplatePath);
        DiscoveryFingerprint = Hash(File.ReadAllText(discoveryTemplatePath));
        RepairFingerprint = Hash(File.ReadAllText(repairTemplatePath) + "\n" + GraphAnswerContract.NoEvidenceAnswer);
    }
 
    public string Fingerprint { get; }
    public string DiscoveryFingerprint { get; }
    public string RepairFingerprint { get; }
 
    public Task<ChatMessage[]> RenderDiscoveryAsync(string question, string discoveryContext,
        CancellationToken cancellationToken = default) =>
        RenderStageAsync(discoveryPrompt, new Dictionary<string, string>
        {
            ["question"] = question,
            ["discovery_context"] = discoveryContext
        }, [ChatRole.System, ChatRole.User], cancellationToken);
 
    public async Task<ChatMessage> RenderCitationRepairAsync(CancellationToken cancellationToken = default) =>
        (await RenderStageAsync(repairPrompt, new Dictionary<string, string>
        {
            ["no_evidence_answer"] = GraphAnswerContract.NoEvidenceAnswer
        }, [ChatRole.User], cancellationToken))[0];
 
    private static string Hash(string text) =>
        Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ReplaceLineEndings("\n"))));
 
    private static async Task<ChatMessage[]> RenderStageAsync(Agent template,
        Dictionary<string, string> values, ChatRole[] roles, CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();
        string prefix = $"__graph_stage_{Guid.NewGuid():N}_";
        var inputs = new Dictionary<string, string>();
        var arguments = new Dictionary<string, object?>();
        foreach (var (name, value) in values)
        {
            string key = $"{prefix}{inputs.Count}__";
            inputs.Add(key, value);
            arguments.Add(name, key);
        }
        var prepared = await Pipeline.PrepareAsync(template, arguments);
        cancellationToken.ThrowIfCancellationRequested();
        if (prepared.Count != roles.Length || prepared.Where((message, index) =>
            message.Role != (roles[index] == ChatRole.System ? Role.System : Role.User)).Any())
            throw new InvalidDataException("The Graph stage template must preserve its expected message roles.");
 
        return prepared.Select((message, index) => new ChatMessage(roles[index],
            Regex.Replace(message.Text, prefix + "[0-9]+__", match => inputs[match.Value]))).ToArray();
    }
 
    public async Task<ChatMessage[]> RenderAsync(
        string question,
        IEnumerable<KeyValuePair<string, string>> documents,
        CancellationToken cancellationToken = default,
        string graphContext = "")
    {
        cancellationToken.ThrowIfCancellationRequested();
        string prefix = $"__graph_input_{Guid.NewGuid():N}_";
        var inputs = new Dictionary<string, string>();
        string Placeholder(string value)
        {
            string key = $"{prefix}{inputs.Count}__";
            inputs.Add(key, value);
            return key;
        }
 
        var prepared = await Pipeline.PrepareAsync(prompt, new Dictionary<string, object?>
        {
            ["question"] = Placeholder(question),
            ["documents"] = documents.Select(document => new Dictionary<string, object?>
            {
                ["name"] = Placeholder(document.Key),
                ["passage"] = Placeholder(document.Value)
            }).ToArray(),
            ["graph_context"] = Placeholder(graphContext),
            ["no_evidence_answer"] = GraphAnswerContract.NoEvidenceAnswer
        });
        cancellationToken.ThrowIfCancellationRequested();
 
        if (prepared.Count != 2 || prepared[0].Role != Role.System || prepared[1].Role != Role.User)
            throw new InvalidDataException("The Graph answer template must produce one system and one user message.");
 
        string userText = Regex.Replace(prepared[1].Text, prefix + "[0-9]+__",
            match => inputs[match.Value]);
        return [new(ChatRole.System, prepared[0].Text), new(ChatRole.User, userText)];
    }
}

using System.Security.Cryptography;
using Microsoft.Extensions.AI;
using Prompty.Core;
 
namespace MCP_Aspire.Prompts;
 
public sealed class GroundedAnswerPromptRenderer
{
    private readonly Agent prompt;
 
    public GroundedAnswerPromptRenderer()
        : this(Path.Combine(AppContext.BaseDirectory, "Vector", "grounded-answer.prompty"))
    {
    }
 
    public GroundedAnswerPromptRenderer(string templatePath)
    {
        NonceSafePromptyChatParser.Register();
        prompt = PromptyLoader.Load(templatePath);
        Fingerprint = Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(templatePath)));
    }
 
    public string Fingerprint { get; }
 
    public async Task<ChatMessage[]> RenderAsync(string context, string question, CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();
        var preparedMessages = await Pipeline.PrepareAsync(prompt, new Dictionary<string, object?>
        {
            ["context"] = context,
            ["question"] = question
        });
        cancellationToken.ThrowIfCancellationRequested();
 
        return preparedMessages
            .Select(message => new ChatMessage(ToChatRole(message.Role), message.Text))
            .ToArray();
    }
 
    private static ChatRole ToChatRole(Role role) => role.ToString().ToLowerInvariant() switch
    {
        "system" => ChatRole.System,
        "assistant" => ChatRole.Assistant,
        "tool" => ChatRole.Tool,
        _ => ChatRole.User
    };
}

using System.Globalization;
using Prompty.Core;
 
namespace MCP_Aspire.Prompts;
 
internal sealed class NonceSafePromptyChatParser : PromptyChatParser, IPreRenderable
{
    private static readonly Lazy<bool> Registration = new(() =>
    {
        _ = new PromptyBuilder();
        InvokerRegistry.RegisterParser("prompty", new NonceSafePromptyChatParser());
        return true;
    });
 
    internal static void Register() => _ = Registration.Value;
 
    (string template, Dictionary<string, object?> context) IPreRenderable.PreRender(string template) =>
        PrepareTemplate(template, base.PreRender);
 
    internal static (string template, Dictionary<string, object?> context) PrepareTemplate(
        string template,
        Func<string, (string template, Dictionary<string, object?> context)> preRender)
    {
        for (int attempt = 0; attempt < 32; attempt++)
        {
            var prepared = preRender(template);
            string nonce = (string)prepared.context["nonce"]!;
            if (!double.TryParse(nonce, NumberStyles.Float, CultureInfo.InvariantCulture, out _))
                return prepared;
        }
 
        throw new InvalidOperationException("Prompty could not generate a nonnumeric role-marker nonce.");
    }
}

using MCP_Aspire.Domain.Graph;
 
namespace MCP_Aspire.RAGApi.Graph.Models;
 
public sealed record GraphDocument(int Version, IReadOnlyDictionary<string, string> SourceHashes,
    IReadOnlyList<GraphNode> Nodes, IReadOnlyList<GraphEdge> Edges, IReadOnlyList<GraphEvidence> Evidence);

using MCP_Aspire.Domain.Graph;
 
namespace MCP_Aspire.RAGApi.Graph.Tools;
 
// All evidence collected during one answer, including items omitted from discovery responses.
internal sealed class GraphAnswerEvidence(GraphSearchResult initialContext)
{
    private readonly Dictionary<string, GraphEvidence> _evidence = initialContext.Evidence.ToDictionary(item => item.Id);
    private readonly Dictionary<string, GraphNode> _nodes = initialContext.Nodes.ToDictionary(node => node.Id);
    private readonly Dictionary<(string From, string Type, string To), GraphEdge> _edges =
        initialContext.Edges.ToDictionary(edge => (edge.From, edge.Type, edge.To));
 
    internal bool Truncated { get; private set; } = initialContext.Truncated;
    internal IReadOnlyList<GraphNode> Nodes => _nodes.Values.OrderBy(node => node.Id, StringComparer.Ordinal).ToArray();
    internal IReadOnlyList<GraphEdge> Edges => _edges.Values
        .OrderBy(edge => edge.From, StringComparer.Ordinal).ThenBy(edge => edge.Type, StringComparer.Ordinal)
        .ThenBy(edge => edge.To, StringComparer.Ordinal).ToArray();
 
    internal IReadOnlyList<GraphEvidence> Evidence => _evidence.Values.OrderBy(item => item.Id, StringComparer.Ordinal).ToArray();
 
    internal void Include(GraphSearchResult result)
    {
        Truncated |= result.Truncated;
        foreach (var node in result.Nodes)
            _nodes.TryAdd(node.Id, node);
        foreach (var edge in result.Edges)
            _edges.TryAdd((edge.From, edge.Type, edge.To), edge);
        foreach (var evidence in result.Evidence)
            _evidence.TryAdd(evidence.Id, evidence);
    }
}

using System.ComponentModel;
using System.Diagnostics;
using MCP_Aspire.Domain.Graph;
using Microsoft.Extensions.AI;
 
namespace MCP_Aspire.RAGApi.Graph.Tools;
 
// Created per answer: evidence and tool budgets never leak between requests.
public sealed class GraphChatTools(GraphQueryService queries, GraphSearchResult initialContext, GenAiTracing tracing = default)
{
    private readonly GraphAnswerEvidence _answerEvidence = new(initialContext);
    private readonly GraphDiscoveryResults _discoveryResults = new(initialContext);
    private int _calls;
 
    public bool Truncated => _answerEvidence.Truncated;
    public IReadOnlyList<GraphNode> UsedNodes => _answerEvidence.Nodes;
    public IReadOnlyList<GraphEdge> UsedEdges => _answerEvidence.Edges;
    public IReadOnlyList<GraphEvidence> UsedEvidence => _answerEvidence.Evidence;
 
    public IList<AITool> CreateTools() =>
    [
        AIFunctionFactory.Create(SearchNodes, new AIFunctionFactoryOptions { MarshalResult = _discoveryResults.MarshalResult }),
        AIFunctionFactory.Create(GetNode, new AIFunctionFactoryOptions { MarshalResult = _discoveryResults.MarshalResult }),
        AIFunctionFactory.Create(GetNeighbors, new AIFunctionFactoryOptions { MarshalResult = _discoveryResults.MarshalResult })
    ];
 
    [Description("Find graph concepts by name or alias. Returns new nodes and one-hop relationships; previously returned items are omitted. Source passages are retained for the final answer. Results may be truncated; absence is not proof of no relationship.")]
    public GraphSearchResult SearchNodes([Description("Concept name or keywords.")] string query,
        [Description("Maximum matching concepts, 1-10.")] int max, CancellationToken cancellationToken) =>
        Execute("SearchNodes", new { query, max }, () => queries.Search(query, max, cancellationToken));
 
    [Description("Read a concept by exact node ID. Previously returned items are omitted; supporting passages are retained for the final answer.")]
    public GraphSearchResult GetNode(string id, CancellationToken cancellationToken) =>
        Execute("GetNode", new { id }, () => queries.GetNode(id, cancellationToken));
 
    [Description("Traverse incoming and outgoing relationships. From and To preserve edge direction. Maximum depth is two hops. Returns new nodes, relationships and a truncation flag; previously returned items are omitted. Supporting passages are retained for the final answer.")]
    public GraphSearchResult GetNeighbors(string id,
        [Description("Relationship type filter, or null for all types.")] string? edgeType,
        [Description("Traversal depth, 1 or 2.")] int depth, CancellationToken cancellationToken) =>
        Execute("GetNeighbors", new { id, edgeType, depth }, () => queries.GetNeighbors(id, edgeType, depth, cancellationToken));
 
    private GraphSearchResult Execute(string name, object input, Func<GraphSearchResult> query)
    {
        if (++_calls > GraphGenerationSettings.MaxToolCalls)
            throw new InvalidOperationException("The graph tool budget for this answer is exhausted.");
        using Activity? activity = GraphTelemetry.StartTool(name, input, tracing);
        GraphSearchResult result = query();
        GraphTelemetry.RecordToolResult(activity, name, result, tracing);
        _answerEvidence.Include(result);
        return result;
    }
}

using System.Text.Json;
using MCP_Aspire.Domain.Graph;
 
namespace MCP_Aspire.RAGApi.Graph.Tools;
 
// Track only what the discovery model has already seen; final evidence is retained separately.
internal sealed class GraphDiscoveryResults(GraphSearchResult initialContext)
{
    private readonly HashSet<string> _reportedNodes = initialContext.Nodes.Select(node => node.Id).ToHashSet();
    private readonly HashSet<(string From, string Type, string To)> _reportedEdges = initialContext.Edges
        .Select(edge => (edge.From, edge.Type, edge.To)).ToHashSet();
 
    // Discovery needs concept IDs and directed relationships. Keep source passages
    // server-side and supply them once in the final grounded answer prompt.
    internal static object Context(GraphSearchResult result) => new
    {
        Nodes = result.Nodes.Select(node => new { node.Id, node.Type, node.Label }),
        Edges = result.Edges.Select(edge => new { edge.From, edge.Type, edge.To }),
        result.Truncated
    };
 
    internal ValueTask<object?> MarshalResult(object? result, Type? resultType, CancellationToken cancellationToken)
    {
        var graph = (GraphSearchResult)result!;
        return ValueTask.FromResult<object?>(JsonSerializer.SerializeToElement(Context(graph with
        {
            Nodes = graph.Nodes.Where(node => _reportedNodes.Add(node.Id)).ToArray(),
            Edges = graph.Edges.Where(edge => _reportedEdges.Add((edge.From, edge.Type, edge.To))).ToArray()
        })));
    }
}

using MCP_Aspire.RAGApi.Shared.Generation;
using Microsoft.Extensions.AI;
 
namespace MCP_Aspire.RAGApi.Graph;
 
internal sealed class ContextLimitedChatClient(IChatClient inner, int contextTokens, int answerTokens) : DelegatingChatClient(inner)
{
    public override Task<ChatResponse> GetResponseAsync(IEnumerable<ChatMessage> messages,
        ChatOptions? options = null, CancellationToken cancellationToken = default)
    {
        ChatMessage[] input = messages.ToArray();
        ChatContextBudget.EnsureFits(input, options, contextTokens, answerTokens);
        return InnerClient.GetResponseAsync(input, options, cancellationToken);
    }
}

using System.Text;
using System.Text.RegularExpressions;
using MCP_Aspire.RAGApi.Graph.Tools;
using MCP_Aspire.Domain.Graph;
 
namespace MCP_Aspire.RAGApi.Graph;
 
internal static class GraphAnswerContextBuilder
{
    internal static (string GraphContext, IEnumerable<KeyValuePair<string, string>> Documents)
        Build(string query, GraphChatTools tools)
    {
        string graphContext = FormatGraphContext(tools);
        var documents = OrderEvidence(query, tools.UsedEvidence);
        return (graphContext, documents);
    }
 
    private static string FormatGraphContext(GraphChatTools tools)
    {
        var prompt = new StringBuilder("Retrieved graph concepts:\n");
        foreach (var node in tools.UsedNodes)
            prompt.AppendLine($"{node.Id}: {node.Label} ({node.Type})");
        prompt.AppendLine("Retrieved graph relationships (From --Type--> To):");
        foreach (var edge in tools.UsedEdges)
            prompt.AppendLine($"{edge.From} --{edge.Type}--> {edge.To}");
        prompt.AppendLine($"Graph results truncated: {tools.Truncated}. Do not assume an exhaustive list.");
        return prompt.ToString();
    }
 
    private static IEnumerable<KeyValuePair<string, string>> OrderEvidence(string query,
        IReadOnlyList<GraphEvidence> evidence)
    {
        // Preserve every retrieved passage, prioritizing those relevant to the question.
        string[] terms = GetPassageTerms(query);
        var documents = evidence.GroupBy(item => item.SourceName, StringComparer.Ordinal)
            .Select(document => new
            {
                SourceName = document.Key,
                Passages = document.OrderByDescending(item => ScorePassage(item.Text, terms)).ThenBy(item => item.Id, StringComparer.Ordinal).ToArray()
            })
            .OrderByDescending(document => document.Passages.Max(item => ScorePassage(item.Text, terms)))
            .ThenBy(document => document.SourceName, StringComparer.Ordinal);
        return documents.Select(document => new KeyValuePair<string, string>(document.SourceName,
            string.Join(Environment.NewLine, document.Passages.Select(item => item.Text))));
    }
 
    private static string[] GetPassageTerms(string query) =>
        Regex.Matches(query.ToLowerInvariant(), "[a-z0-9]+")
            .Select(match => match.Value)
            .Where(term => term.Length >= 3 && term is not ("dapr" or "what" or "which" or "does" or "how" or "the" or "can" or "and" or "are" or "for" or "with" or "from" or "through" or "another" or "these" or "that"))
            .Select(term => term[..Math.Min(term.Length, 6)]).Distinct(StringComparer.Ordinal).ToArray();
 
    private static int ScorePassage(string text, string[] terms) =>
        terms.Count(term => text.Contains(term, StringComparison.OrdinalIgnoreCase));
}

using System.Diagnostics;
using System.Text.Json;
using MCP_Aspire.Domain.Graph;
using MCP_Aspire.Prompts;
using MCP_Aspire.RAGApi.Graph.Tools;
using MCP_Aspire.RAGApi.Shared.Configuration;
using MCP_Aspire.RAGApi.Shared.Generation;
using MCP_Aspire.RAGApi.Shared.Telemetry;
using MCP_Aspire.RAGApi.Shared.Validation;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Options;
 
namespace MCP_Aspire.RAGApi.Graph;
 
public sealed class GraphAnswerService(
    GraphQueryService queries, IChatClient chatClient, RAGApiTelemetry telemetry,
    IConfiguration configuration, ILogger<GraphAnswerService> logger, IOptions<RagOptions> settings,
    GraphAnswerPromptRenderer promptRenderer)
{
    public const string NoEvidenceAnswer = GraphAnswerContract.NoEvidenceAnswer;
 
    public async Task<string> AskAsync(string query, CancellationToken cancellationToken) =>
        (await AskWithDiagnosticsAsync(query, cancellationToken)).Answer;
 
    public Task<GraphAnswerResult> AskWithDiagnosticsAsync(string query, CancellationToken cancellationToken) =>
        AnswerDeadline.RunAsync(settings.Value.AnswerTimeoutSeconds, cancellationToken,
            token => AskCoreAsync(query, token));
 
    private async Task<GraphAnswerResult> AskCoreAsync(string query, CancellationToken cancellationToken)
    {
        QueryValidation.Validate(query, settings.Value);
        cancellationToken.ThrowIfCancellationRequested();
        var tracing = GenAiTracing.FromConfiguration(configuration);
        var model = ModelConnectionOptions.FromConnectionString(configuration, "chat");
        AnswerRunMetadata run = CreateRunMetadata(model.ModelId);
        using Activity? activity = GraphTelemetry.StartRequest("ragapi.graph.ask", "Graph answer", query, tracing);
        GraphSearchResult context = RetrieveEvidence(query, cancellationToken);
        if (context.Nodes.Count == 0)
        {
            GraphTelemetry.RecordNoEvidence(activity, NoEvidenceAnswer, tracing);
            return new(NoEvidenceAnswer, run);
        }
 
        var tools = new GraphChatTools(queries, context, tracing);
        var options = ChatOptionsFactory.CreateOptions(settings.Value, graph: true, tools.CreateTools());
        var (tracedClient, limitedClient, discoveryClient) = CreateAnswerClients(model.ModelId, tracing);
        if (settings.Value.GraphEnableTools)
        {
            await DiscoverEvidenceAsync(discoveryClient, query, context, options, cancellationToken);
        }
        var (graphContext, documents) = GraphAnswerContextBuilder.Build(query, tools);
        ChatMessage[] answerMessages = await promptRenderer.RenderAsync(query, documents,
            cancellationToken, graphContext);
        ChatOptions answerOptions = GraphGenerationSettings.CreateAnswerOptions(settings.Value);
        var response = await limitedClient.GetResponseAsync(answerMessages, answerOptions, cancellationToken);
        string text = ValidateAnswer(response, tools);
        if (NeedsCitationRepair(text, response.Text))
        {
            response = await RepairCitationsAsync(limitedClient, answerMessages, response, answerOptions, cancellationToken);
            text = ValidateAnswer(response, tools);
        }
        string answer = FormatAnswer(text);
        GraphTelemetry.RecordAnswer(activity, tools.UsedEvidence.Count, answer, tracing);
        return new(answer, run with
        {
            ModelCalls = tracedClient.Calls,
            FinishReason = response.FinishReason?.ToString(),
            InputTokens = response.Usage?.InputTokenCount,
            OutputTokens = response.Usage?.OutputTokenCount
        });
    }
 
    private AnswerRunMetadata CreateRunMetadata(string modelId)
    {
        return new AnswerRunMetadata(
            Model: modelId,
            PromptFingerprint: promptRenderer.Fingerprint,
            MaxAnswerTokens: settings.Value.MaxAnswerTokens,
            ContextTokens: settings.Value.GraphAnswerContextTokens,
            Temperature: GraphGenerationSettings.Temperature,
            Seed: GraphGenerationSettings.Seed,
            Variant: settings.Value.GraphEnableTools ? "graph-tools" : "graph")
        {
            ContextPolicy = GraphGenerationSettings.ContextPolicy,
            DiscoveryPromptFingerprint = settings.Value.GraphEnableTools ? promptRenderer.DiscoveryFingerprint : null,
            RepairPromptFingerprint = promptRenderer.RepairFingerprint
        };
    }
 
    private GraphSearchResult RetrieveEvidence(string query, CancellationToken cancellationToken)
    {
        long started = Stopwatch.GetTimestamp();
        GraphSearchResult context = queries.Search(query, cancellationToken: cancellationToken);
        telemetry.RecordSearch("graph.ask", "ok", context.Nodes.Count, Stopwatch.GetElapsedTime(started));
        return context;
    }
 
    private (GraphModelChatClient Traced, ContextLimitedChatClient Limited, FunctionInvokingChatClient Discovery)
        CreateAnswerClients(string modelId, GenAiTracing tracing)
    {
        var tracedClient = new GraphModelChatClient(chatClient, modelId, telemetry, tracing, logger);
        var limitedClient = new ContextLimitedChatClient(tracedClient, settings.Value.GraphAnswerContextTokens, settings.Value.MaxAnswerTokens);
        // Only the Graph flow uses this wrapper. DI retains ownership of the shared inner client.
        var client = new FunctionInvokingChatClient(limitedClient)
        {
            MaximumIterationsPerRequest = GraphGenerationSettings.MaxDiscoveryIterations,
            AllowConcurrentInvocation = false
        };
        return (tracedClient, limitedClient, client);
    }
 
    private async Task DiscoverEvidenceAsync(IChatClient client, string query, GraphSearchResult context,
        ChatOptions options, CancellationToken cancellationToken)
    {
        ChatMessage[] messages = await promptRenderer.RenderDiscoveryAsync(query,
            JsonSerializer.Serialize(GraphDiscoveryResults.Context(context)), cancellationToken);
        await client.GetResponseAsync(messages, options, cancellationToken);
    }
 
    private static string ValidateAnswer(ChatResponse response, GraphChatTools tools) =>
        AnswerCitationPolicy.ValidateCitations(response.Text ?? string.Empty,
            tools.UsedEvidence.Select(item => item.SourceName), NoEvidenceAnswer);
 
    private static bool NeedsCitationRepair(string validatedAnswer, string? draft) =>
        validatedAnswer == NoEvidenceAnswer && !string.IsNullOrWhiteSpace(draft) && draft.Trim() != NoEvidenceAnswer;
 
    private async Task<ChatResponse> RepairCitationsAsync(IChatClient client, ChatMessage[] answerMessages,
        ChatResponse rejectedDraft, ChatOptions options, CancellationToken cancellationToken)
    {
        // One repair attempt; the caller validates its citations before accepting it.
        ChatMessage[] repairMessages =
        [
            .. answerMessages,
            new(ChatRole.Assistant, rejectedDraft.Text),
            await promptRenderer.RenderCitationRepairAsync(cancellationToken)
        ];
        return await client.GetResponseAsync(repairMessages, options, cancellationToken);
    }
 
    private static string FormatAnswer(string text)
    {
        string sources = string.Join(Environment.NewLine, AnswerCitationPolicy.CitedSources(text)
            .Order(StringComparer.Ordinal).Select(source => $"[source: {source}]"));
        return text == NoEvidenceAnswer ? text : $"{text}\n\nSources:\n{sources}";
    }
}

using System.Diagnostics;
using System.Text.Json;
using MCP_Aspire.Domain.Graph;
using MCP_Aspire.Domain;
using MCP_Aspire.RAGApi.Shared.Telemetry;
using MCP_Aspire.RAGApi.Shared.Validation;
 
namespace MCP_Aspire.RAGApi.Graph;
 
public static class GraphEndpoints
{
    public static RouteGroupBuilder MapGraphEndpoints(this RouteGroupBuilder api)
    {
        var graph = api.MapGroup("/graph");
        graph.AddEndpointFilter<QueryValidationFilter>();
        graph.AddEndpointFilter((context, next) =>
        {
            GraphTelemetry.TagRetrievalMode(Activity.Current);
            return next(context);
        });
        graph.MapPost("/search", Search);
        graph.MapPost("/ask", AskAsync);
        return graph;
    }
 
    private static IResult Search(SearchRequest request, GraphQueryService queries,
        RAGApiTelemetry telemetry, IConfiguration configuration, CancellationToken cancellationToken)
    {
        var tracing = GenAiTracing.FromConfiguration(configuration);
        using Activity? activity = GraphTelemetry.StartRequest("ragapi.graph.search", "Graph search", request.Query, tracing);
        long started = Stopwatch.GetTimestamp();
 
        var result = queries.Search(request.Query, cancellationToken: cancellationToken);
 
        GraphTelemetry.RecordSearchResult(activity, result, tracing);
        telemetry.RecordSearch("graph.search", "ok", result.Nodes.Count, Stopwatch.GetElapsedTime(started));
 
        return Results.Ok(result);
    }
 
    private static async Task<IResult> AskAsync(SearchRequest request, GraphAnswerService answers,
        HttpResponse response, CancellationToken cancellationToken)
    {
        var result = await answers.AskWithDiagnosticsAsync(request.Query, cancellationToken);
        response.Headers[AnswerRunMetadata.HeaderName] = JsonSerializer.Serialize(result.Run);
        return Results.Ok(result.Answer);
    }
}

using MCP_Aspire.RAGApi.Shared.Configuration;
using MCP_Aspire.RAGApi.Shared.Generation;
using Microsoft.Extensions.AI;
 
namespace MCP_Aspire.RAGApi.Graph;
 
internal static class GraphGenerationSettings
{
    internal const float Temperature = 0;
    internal const long Seed = 42;
    internal const int MaxDiscoveryIterations = 6;
    internal const int MaxToolCalls = 12;
    internal const string ContextPolicy = "all-retrieved-passages-v1";
 
    internal static ChatOptions CreateAnswerOptions(RagOptions options)
    {
        ChatOptions chat = ChatOptionsFactory.CreateOptions(options, graph: true);
        chat.Temperature = Temperature;
        chat.Seed = Seed;
        return chat;
    }
}

using MCP_Aspire.Prompts;
 
namespace MCP_Aspire.RAGApi.Graph;
 
internal sealed class GraphInitialization(IServiceProvider services) : IHostedService
{
    public Task StartAsync(CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();
        _ = services.GetRequiredService<GraphStore>();
        _ = services.GetRequiredService<GraphAnswerPromptRenderer>();
        return Task.CompletedTask;
    }
 
    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}

using System.Diagnostics;
using MCP_Aspire.RAGApi.Shared.Telemetry;
using Microsoft.Extensions.AI;
 
namespace MCP_Aspire.RAGApi.Graph;
 
// Trace each provider call, including every iteration of tool exploration.
internal sealed class GraphModelChatClient(IChatClient inner, string modelId, RAGApiTelemetry telemetry,
    GenAiTracing tracing, ILogger logger) : DelegatingChatClient(inner)
{
    public int Calls { get; private set; }
    public override Task<ChatResponse> GetResponseAsync(IEnumerable<ChatMessage> messages,
        ChatOptions? options = null, CancellationToken cancellationToken = default) =>
        InvokeAsync(messages.ToArray(), options, cancellationToken);
 
    private async Task<ChatResponse> InvokeAsync(ChatMessage[] messages, ChatOptions? options, CancellationToken cancellationToken)
    {
        Calls++;
        bool tools = options?.Tools?.Count > 0;
        using Activity? activity = RAGApiTelemetry.StartActivity(tools ? "ragapi.graph.model.tools" : "ragapi.graph.model.chat", "LLM", "LLM");
        activity.SetModelRequest("chat", "ollama", modelId);
        tracing.CaptureChatInput(activity, messages.Select(message => (message.Role.ToString(), message.Text)));
        return await telemetry.InvokeChatAsync(tools ? "graph.tools" : "graph.chat", activity,
            messages.Sum(message => message.Text?.Length ?? 0), tracing, logger,
            () => InnerClient.GetResponseAsync(messages, options, cancellationToken));
    }
}

using System.Text.RegularExpressions;
using MCP_Aspire.Domain.Graph;
 
namespace MCP_Aspire.RAGApi.Graph;
 
public sealed partial class GraphQueryService(GraphStore store)
{
    public const int MaxNodes = 30;
    public const int MaxEdges = 40;
    private static readonly HashSet<string> StopWords = new(StringComparer.Ordinal)
    {
        "a", "an", "and", "are", "as", "at", "be", "by", "can", "dapr", "do", "does", "for", "from",
        "how", "i", "in", "is", "it", "of", "on", "or", "that", "the", "these", "this", "to", "use",
        "uses", "using", "what", "when", "where", "which", "who", "with"
    };
 
    public GraphSearchResult Search(string query, int max = 5, CancellationToken cancellationToken = default)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(query);
        cancellationToken.ThrowIfCancellationRequested();
 
        var matches = FindMatchingConcepts(query);
        int limit = Math.Clamp(max, 1, 10);
 
        return Traverse(matches.Take(limit).Select(item => item.Node.Id), depth: 1, edgeType: null,
            truncated: matches.Length > limit, cancellationToken);
    }
 
    public GraphSearchResult GetNode(string id, CancellationToken cancellationToken = default) =>
        Traverse([id], 0, null, false, cancellationToken);
 
    public GraphSearchResult GetNeighbors(string id, string? edgeType = null, int depth = 1,
        CancellationToken cancellationToken = default) =>
        Traverse([id], Math.Clamp(depth, 1, 2), edgeType, false, cancellationToken);
 
    private GraphSearchResult Traverse(IEnumerable<string> seeds, int depth, string? edgeType, bool truncated,
        CancellationToken cancellationToken)
    {
        cancellationToken.ThrowIfCancellationRequested();
        var nodes = seeds.Where(store.Nodes.ContainsKey).Distinct(StringComparer.Ordinal).ToList();
        var visited = nodes.ToHashSet(StringComparer.Ordinal);
        var edges = new List<GraphEdge>();
        string[] frontier = nodes.ToArray();
        for (int hop = 0; hop < depth; hop++)
        {
            var next = new List<string>();
            foreach (string id in frontier)
            {
                foreach (var edge in store.GetEdges(id))
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    if (edgeType is not null && !string.Equals(edge.Type, edgeType, StringComparison.OrdinalIgnoreCase))
                        continue;
                    if (edges.Contains(edge))
                        continue;
                    string neighbor = edge.From == id ? edge.To : edge.From;
                    if (edges.Count >= MaxEdges || (!visited.Contains(neighbor) && nodes.Count >= MaxNodes))
                    {
                        truncated = true;
                        continue;
                    }
                    edges.Add(edge);
                    if (visited.Add(neighbor))
                    {
                        nodes.Add(neighbor);
                        next.Add(neighbor);
                    }
                }
            }
            frontier = next.ToArray();
        }
 
        GraphNode[] resultNodes = nodes.Select(id => store.Nodes[id]).ToArray();
        GraphEvidence[] evidence = CollectEvidence(resultNodes, edges);
        return new GraphSearchResult(resultNodes, edges, evidence, truncated);
    }
 
    private (GraphNode Node, int Score)[] FindMatchingConcepts(string query)
    {
        string normalized = Normalize(query);
 
        string[] terms = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries)
            .Where(term => !StopWords.Contains(term)).Distinct().ToArray();
 
        return store.Nodes.Values.Select(node => (Node: node, Score: Score(node, normalized, terms)))
            .Where(item => item.Score > 0).OrderByDescending(item => item.Score)
            .ThenBy(item => item.Node.Id, StringComparer.Ordinal).ToArray();
    }
 
    private GraphEvidence[] CollectEvidence(IEnumerable<GraphNode> nodes, IEnumerable<GraphEdge> edges) =>
        nodes.SelectMany(node => node.EvidenceIds)
            .Concat(edges.SelectMany(edge => edge.EvidenceIds)).Distinct(StringComparer.Ordinal)
            .Select(id => store.Evidence[id]).OrderBy(item => item.Id, StringComparer.Ordinal).ToArray();
 
    private static int Score(GraphNode node, string query, string[] terms)
    {
        string[] names = node.Aliases.Prepend(node.Label).Select(Normalize).ToArray();
        if (terms.Length == 0)
            return node.Id == "dapr" && query.Split(' ').Contains("dapr") ? 1 : 0;
        int overlap = names.Max(name => terms.Count(term => name.Split(' ').Contains(term)));
        if (overlap == 0)
            return 0;
        return overlap + (names.Any(name => $" {query} ".Contains($" {name} ", StringComparison.Ordinal)) ? 10 : 0);
    }
 
    private static string Normalize(string value) => Words().Replace(value.ToLowerInvariant(), " ").Trim();
 
    [GeneratedRegex("[^a-z0-9]+")]
    private static partial Regex Words();
}

using MCP_Aspire.Prompts;
 
namespace MCP_Aspire.RAGApi.Graph;
 
public static class GraphServiceRegistration
{
    public static IServiceCollection AddKnowledgeGraph(this IServiceCollection services)
    {
        services.AddSingleton(sp => GraphStore.Load(sp.GetRequiredService<IHostEnvironment>().ContentRootPath));
        services.AddHostedService<GraphInitialization>();
        services.AddSingleton<GraphQueryService>();
        services.AddSingleton<GraphAnswerPromptRenderer>();
        services.AddScoped<GraphAnswerService>();
        return services;
    }
}

using System.Security.Cryptography;
using System.Text.Json;
using MCP_Aspire.Domain.Graph;
using MCP_Aspire.RAGApi.Graph.Models;
 
namespace MCP_Aspire.RAGApi.Graph;
 
// This snapshot loads independently of DocumentLoading, DocumentMemory and embeddings.
public sealed class GraphStore
{
    public IReadOnlyDictionary<string, GraphNode> Nodes { get; }
    public IReadOnlyDictionary<string, GraphEvidence> Evidence { get; }
    private readonly Dictionary<string, GraphEdge[]> _adjacency;
 
    private GraphStore(GraphDocument document)
    {
        Nodes = document.Nodes.ToDictionary(node => node.Id, StringComparer.Ordinal);
        Evidence = document.Evidence.ToDictionary(item => item.Id, StringComparer.Ordinal);
        _adjacency = document.Edges.SelectMany(edge => new[] { (Id: edge.From, Edge: edge), (Id: edge.To, Edge: edge) })
            .GroupBy(item => item.Id, StringComparer.Ordinal)
            .ToDictionary(group => group.Key, group => group.Select(item => item.Edge).Distinct().ToArray(), StringComparer.Ordinal);
    }
 
    public IReadOnlyList<GraphEdge> GetEdges(string id) => _adjacency.GetValueOrDefault(id) ?? [];
 
    public static GraphStore Load(string contentRoot)
    {
        string dataPath = Path.Combine(contentRoot, "SourceCorpus");
        string graphPath = Path.Combine(contentRoot, "KnowledgeGraph", "dapr-graph.json");
        var document = JsonSerializer.Deserialize<GraphDocument>(File.ReadAllText(graphPath),
            new JsonSerializerOptions(JsonSerializerDefaults.Web))
            ?? throw new InvalidDataException("The Dapr graph snapshot is empty.");
        Validate(document, dataPath);
        return new GraphStore(document);
    }
 
    private static void Validate(GraphDocument document, string dataPath)
    {
        ValidateSchema(document);
        var sourceTexts = ValidateSourceManifest(document.SourceHashes, dataPath);
        var evidenceIds = ValidateSourceEvidence(document.Evidence, sourceTexts);
        var nodeIds = ValidateNodes(document.Nodes, evidenceIds);
        ValidateEdges(document.Edges, nodeIds, evidenceIds);
    }
 
    private static void ValidateSchema(GraphDocument document)
    {
        if (document.Version != 1 || document.Nodes is null || document.Edges is null
            || document.Evidence is null || document.SourceHashes is null)
            throw new InvalidDataException("The Dapr graph snapshot has an unsupported schema.");
    }
 
    private static Dictionary<string, string> ValidateSourceManifest(
        IReadOnlyDictionary<string, string> sourceHashes, string dataPath)
    {
        var sourceTexts = new Dictionary<string, string>(StringComparer.Ordinal);
        foreach (var (source, expectedHash) in sourceHashes)
        {
            if (Path.GetFileName(source) != source || !source.EndsWith(".md", StringComparison.Ordinal))
                throw new InvalidDataException($"Invalid graph source: {source}.");
            string path = Path.Combine(dataPath, source);
            string actualHash = Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
            if (!string.Equals(actualHash, expectedHash, StringComparison.Ordinal))
                throw new InvalidDataException($"Graph source {source} changed. Review and rebuild KnowledgeGraph/dapr-graph.json.");
            sourceTexts.Add(source, File.ReadAllText(path));
        }
 
        if (!sourceTexts.Keys.Order(StringComparer.Ordinal).SequenceEqual(
            Directory.GetFiles(dataPath, "*.md").Select(Path.GetFileName).Order(StringComparer.Ordinal)))
            throw new InvalidDataException("The graph source manifest does not match the markdown corpus.");
        return sourceTexts;
    }
 
    private static HashSet<string> ValidateSourceEvidence(IEnumerable<GraphEvidence> passages,
        IReadOnlyDictionary<string, string> sourceTexts)
    {
        var evidenceIds = new HashSet<string>(StringComparer.Ordinal);
        foreach (var evidence in passages)
        {
            if (string.IsNullOrWhiteSpace(evidence.Id) || !evidenceIds.Add(evidence.Id)
                || string.IsNullOrWhiteSpace(evidence.Text)
                || !sourceTexts.TryGetValue(evidence.SourceName, out string? source)
                || !source.Contains(evidence.Text, StringComparison.Ordinal))
                throw new InvalidDataException($"Invalid or unsupported graph evidence: {evidence.Id}.");
        }
        return evidenceIds;
    }
 
    private static HashSet<string> ValidateNodes(IEnumerable<GraphNode> nodes, HashSet<string> evidenceIds)
    {
        var nodeIds = new HashSet<string>(StringComparer.Ordinal);
        foreach (var node in nodes)
        {
            if (string.IsNullOrWhiteSpace(node.Id) || !nodeIds.Add(node.Id)
                || string.IsNullOrWhiteSpace(node.Label) || string.IsNullOrWhiteSpace(node.Type)
                || node.Aliases is null || node.Aliases.Any(string.IsNullOrWhiteSpace))
                throw new InvalidDataException($"Invalid or duplicate graph node: {node.Id}.");
            ValidateEvidenceReferences(node.EvidenceIds, evidenceIds);
        }
        return nodeIds;
    }
 
    private static void ValidateEdges(IEnumerable<GraphEdge> edges, HashSet<string> nodeIds,
        HashSet<string> evidenceIds)
    {
        var edgeIds = new HashSet<(string, string, string)>();
        foreach (var edge in edges)
        {
            if (!nodeIds.Contains(edge.From) || !nodeIds.Contains(edge.To)
                || string.IsNullOrWhiteSpace(edge.Type) || !edgeIds.Add((edge.From, edge.Type, edge.To)))
                throw new InvalidDataException($"Invalid or duplicate graph edge: {edge.From} {edge.Type} {edge.To}.");
            ValidateEvidenceReferences(edge.EvidenceIds, evidenceIds);
        }
    }
 
    private static void ValidateEvidenceReferences(IReadOnlyList<string>? ids, HashSet<string> evidenceIds)
    {
        if (ids is null || ids.Count == 0 || ids.Any(id => !evidenceIds.Contains(id)))
            throw new InvalidDataException("Every graph node and edge must reference valid source evidence.");
    }
}

using System.Diagnostics;
using MCP_Aspire.Domain.Graph;
using MCP_Aspire.RAGApi.Shared.Telemetry;
 
namespace MCP_Aspire.RAGApi.Graph;
 
internal static class GraphTelemetry
{
    internal static void TagRetrievalMode(Activity? activity) =>
        activity?.SetTag("ragapi.retrieval.mode", "Graph");
 
    internal static Activity? StartRequest(string name, string displayName, string query, GenAiTracing tracing)
    {
        Activity? activity = RAGApiTelemetry.StartRequest(name, displayName, query, tracing);
        TagRetrievalMode(activity);
        return activity;
    }
 
    internal static void RecordNoEvidence(Activity? activity, string answer, GenAiTracing tracing)
    {
        activity?.SetTag("ragapi.outcome", "no_graph_evidence");
        RAGApiTelemetry.CaptureResponse(activity, answer, tracing);
    }
 
    internal static void RecordAnswer(Activity? activity, int evidenceCount, string answer, GenAiTracing tracing)
    {
        activity?.SetTag("ragapi.graph.evidence_count", evidenceCount);
        RAGApiTelemetry.CaptureResponse(activity, answer, tracing);
    }
 
    internal static void RecordSearchResult(Activity? activity, GraphSearchResult result, GenAiTracing tracing)
    {
        activity?.SetTag("ragapi.graph.node_count", result.Nodes.Count);
        activity?.SetTag("ragapi.graph.edge_count", result.Edges.Count);
        activity?.SetTag("ragapi.graph.truncated", result.Truncated);
        tracing.CaptureTraceInputOutput(activity, null, result);
        CaptureEvidence(activity, result, tracing);
        int sourceCount = result.Evidence.Select(item => item.SourceName).Distinct(StringComparer.Ordinal).Count();
        RAGApiTelemetry.CaptureResponsePreview(activity,
            $"{result.Nodes.Count} nodes, {result.Edges.Count} edges, {sourceCount} sources; truncated: {(result.Truncated ? "yes" : "no")}", tracing);
    }
 
    internal static Activity? StartTool(string name, object input, GenAiTracing tracing)
    {
        Activity? activity = RAGApiTelemetry.StartActivity("ragapi.graph.tool", name, "TOOL");
        tracing.CaptureSpanInputOutput(activity, input, null);
        return activity;
    }
 
    internal static void RecordToolResult(Activity? activity, string name, GraphSearchResult result, GenAiTracing tracing)
    {
        tracing.CaptureSpanInputOutput(activity, null, result);
        CaptureEvidence(activity, result, tracing);
        activity?.SetTag("ragapi.graph.tool.name", name);
        activity?.SetTag("ragapi.graph.edge_count", result.Edges.Count);
        activity?.SetTag("ragapi.graph.truncated", result.Truncated);
    }
 
    private static void CaptureEvidence(Activity? activity, GraphSearchResult result, GenAiTracing tracing) =>
        tracing.CaptureRetrievalDocuments(activity, result.Evidence.Select(evidence =>
            (evidence.Text, evidence.SourceName, evidence.Id, (float?)null)));
}