Week 4 Day 24：Eval Runner + CI集成 - 让Agent质量不退化 ⚠️#

// cmd/evalrunner/main.go
package main

import (
    "context"
    "encoding/json"
    "flag"
    "fmt"
    "os"
    "sync"
    "time"

    "github.com/yourname/agent-runtime/internal/agent"
    "github.com/yourname/agent-runtime/internal/eval"
)

func main() {
    var (
        datasetPath   = flag.String("dataset", "evals/dataset.jsonl", "eval dataset path")
        outputPath    = flag.String("out", "report.json", "output report path")
        threshold     = flag.Float64("threshold", 0.85, "pass rate threshold")
        concurrency   = flag.Int("concurrency", 5, "concurrent cases")
        tagFilter     = flag.String("tags", "", "filter by tag (comma-separated)")
        mock          = flag.Bool("mock", false, "use mock LLM")
    )
    flag.Parse()

    // 1. 加载数据集
    cases, err := eval.LoadDataset(*datasetPath)
    if err != nil {
        fmt.Fprintf(os.Stderr, "load dataset: %v\n", err)
        os.Exit(1)
    }

    if *tagFilter != "" {
        cases = eval.FilterByTags(cases, *tagFilter)
    }
    fmt.Printf("Running %d cases with concurrency=%d\n", len(cases), *concurrency)

    // 2. 初始化agent
    ag, err := agent.New(agent.Config{Mock: *mock})
    if err != nil {
        fmt.Fprintf(os.Stderr, "init agent: %v\n", err)
        os.Exit(1)
    }

    // 3. 并发执行
    results := runConcurrent(ag, cases, *concurrency)

    // 4. 聚合报告
    report := eval.Aggregate(results)
    report.Threshold = *threshold
    report.StartTime = time.Now()

    // 5. 写报告
    if err := writeReport(report, *outputPath); err != nil {
        fmt.Fprintf(os.Stderr, "write report: %v\n", err)
        os.Exit(1)
    }

    // 6. 根据阈值决定退出码（给CI用）
    fmt.Printf("\n===== Summary =====\n")
    fmt.Printf("Total:       %d\n", report.Total)
    fmt.Printf("Passed:      %d (%.1f%%)\n", report.Passed, report.PassRate*100)
    fmt.Printf("Failed:      %d\n", report.Failed)
    fmt.Printf("AvgLatency:  %.0fms\n", report.AvgLatencyMs)
    fmt.Printf("TotalCost:   $%.4f\n", report.TotalCostUSD)

    if report.PassRate < *threshold {
        fmt.Printf("\n❌ FAILED: pass rate %.2f < threshold %.2f\n", report.PassRate, *threshold)
        os.Exit(1)
    }
    fmt.Printf("\n✅ PASSED\n")
}

func runConcurrent(ag *agent.Agent, cases []eval.Case, concurrency int) []eval.Result {
    sem := make(chan struct{}, concurrency)
    results := make([]eval.Result, len(cases))
    var wg sync.WaitGroup

    for i, c := range cases {
        wg.Add(1)
        sem <- struct{}{}
        go func(idx int, cs eval.Case) {
            defer wg.Done()
            defer func() { <-sem }()

            start := time.Now()
            ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
            defer cancel()

            resp, err := ag.HandleRequest(ctx, agent.Request{
                UserID:  "eval",
                Message: cs.Input,
            })

            res := eval.Result{
                CaseID:    cs.ID,
                Input:     cs.Input,
                Expected:  cs.ExpectedAnswer,
                Actual:    resp.Answer,
                LatencyMs: time.Since(start).Milliseconds(),
                Tags:      cs.Tags,
            }

            if err != nil {
                res.Passed = false
                res.Error = err.Error()
            } else {
                // 调judge打分
                score, reason := eval.Judge(ctx, cs, resp)
                res.Score = score
                res.Reason = reason
                res.Passed = score >= 0.7  // 0.7为pass阈值
                res.CostUSD = resp.CostUSD
            }

            results[idx] = res
            fmt.Printf("[%d/%d] %s: %s (score=%.2f, %dms)\n",
                idx+1, len(results), cs.ID, passStatus(res.Passed), res.Score, res.LatencyMs)
        }(i, c)
    }

    wg.Wait()
    return results
}

func passStatus(p bool) string {
    if p { return "PASS" }
    return "FAIL"
}

// internal/eval/judge.go
package eval

import (
    "context"
    "encoding/json"
    "fmt"

    "github.com/yourname/agent-runtime/internal/llm"
)

const judgePrompt = `你是eval判分员。对比expected和actual answer，从以下维度打分：
- correctness（是否回答正确）：0-1
- completeness（是否信息完整）：0-1
- relevance（是否切题）：0-1

总分 = 加权平均。严格但公正。

Input: %s
Expected: %s
Actual: %s

输出JSON：{"score": 0.x, "reason": "..."}`

type JudgeOutput struct {
    Score  float64 `json:"score"`
    Reason string  `json:"reason"`
}

func Judge(ctx context.Context, cs Case, actual agent.Response) (float64, string) {
    judge := llm.NewClient("gpt-4o")  // judge用强模型

    prompt := fmt.Sprintf(judgePrompt, cs.Input, cs.ExpectedAnswer, actual.Answer)

    resp, err := judge.Chat(ctx, []llm.Message{
        {Role: "user", Content: prompt},
    }, llm.WithResponseFormat("json_object"))
    if err != nil {
        return 0, fmt.Sprintf("judge error: %v", err)
    }

    var out JudgeOutput
    if err := json.Unmarshal([]byte(resp.Content), &out); err != nil {
        return 0, "judge parse error"
    }

    return out.Score, out.Reason
}

// internal/eval/report.go
package eval

import "time"

type Result struct {
    CaseID    string   `json:"case_id"`
    Input     string   `json:"input"`
    Expected  string   `json:"expected"`
    Actual    string   `json:"actual"`
    Passed    bool     `json:"passed"`
    Score     float64  `json:"score"`
    Reason    string   `json:"reason"`
    LatencyMs int64    `json:"latency_ms"`
    CostUSD   float64  `json:"cost_usd"`
    Error     string   `json:"error,omitempty"`
    Tags      []string `json:"tags"`
}

type Report struct {
    StartTime     time.Time         `json:"start_time"`
    Total         int               `json:"total"`
    Passed        int               `json:"passed"`
    Failed        int               `json:"failed"`
    PassRate      float64           `json:"pass_rate"`
    Threshold     float64           `json:"threshold"`
    AvgScore      float64           `json:"avg_score"`
    AvgLatencyMs  float64           `json:"avg_latency_ms"`
    P95LatencyMs  int64             `json:"p95_latency_ms"`
    TotalCostUSD  float64           `json:"total_cost_usd"`
    ByTag         map[string]TagStat `json:"by_tag"`
    Results       []Result          `json:"results"`
}

type TagStat struct {
    Total   int     `json:"total"`
    Passed  int     `json:"passed"`
    PassRate float64 `json:"pass_rate"`
}

func Aggregate(results []Result) Report {
    r := Report{
        Total:   len(results),
        Results: results,
        ByTag:   make(map[string]TagStat),
    }

    var totalScore, totalLatency, totalCost float64
    latencies := make([]int64, 0, len(results))

    for _, res := range results {
        if res.Passed {
            r.Passed++
        }
        totalScore += res.Score
        totalLatency += float64(res.LatencyMs)
        totalCost += res.CostUSD
        latencies = append(latencies, res.LatencyMs)

        for _, tag := range res.Tags {
            stat := r.ByTag[tag]
            stat.Total++
            if res.Passed {
                stat.Passed++
            }
            r.ByTag[tag] = stat
        }
    }

    r.Failed = r.Total - r.Passed
    if r.Total > 0 {
        r.PassRate = float64(r.Passed) / float64(r.Total)
        r.AvgScore = totalScore / float64(r.Total)
        r.AvgLatencyMs = totalLatency / float64(r.Total)
    }
    r.TotalCostUSD = totalCost

    for tag, stat := range r.ByTag {
        if stat.Total > 0 {
            stat.PassRate = float64(stat.Passed) / float64(stat.Total)
        }
        r.ByTag[tag] = stat
    }

    r.P95LatencyMs = percentile(latencies, 0.95)
    return r
}

func percentile(nums []int64, p float64) int64 {
    if len(nums) == 0 { return 0 }
    sorted := make([]int64, len(nums))
    copy(sorted, nums)
    sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
    idx := int(float64(len(sorted)) * p)
    if idx >= len(sorted) { idx = len(sorted) - 1 }
    return sorted[idx]
}

// internal/eval/markdown.go
package eval

import (
    "fmt"
    "strings"
)

func (r Report) ToMarkdown() string {
    var b strings.Builder

    status := "✅ PASS"
    if r.PassRate < r.Threshold {
        status = "❌ FAIL"
    }

    b.WriteString(fmt.Sprintf("## Eval Report %s\n\n", status))
    b.WriteString(fmt.Sprintf("**Pass rate**: %.1f%% (%d/%d), threshold %.1f%%\n\n",
        r.PassRate*100, r.Passed, r.Total, r.Threshold*100))

    b.WriteString("| Metric | Value |\n|---|---|\n")
    b.WriteString(fmt.Sprintf("| Avg score | %.2f |\n", r.AvgScore))
    b.WriteString(fmt.Sprintf("| Avg latency | %.0fms |\n", r.AvgLatencyMs))
    b.WriteString(fmt.Sprintf("| p95 latency | %dms |\n", r.P95LatencyMs))
    b.WriteString(fmt.Sprintf("| Total cost | $%.4f |\n\n", r.TotalCostUSD))

    // 按tag拆分
    if len(r.ByTag) > 0 {
        b.WriteString("### Breakdown by tag\n\n")
        b.WriteString("| Tag | Pass rate |\n|---|---|\n")
        for tag, stat := range r.ByTag {
            b.WriteString(fmt.Sprintf("| %s | %.1f%% (%d/%d) |\n",
                tag, stat.PassRate*100, stat.Passed, stat.Total))
        }
        b.WriteString("\n")
    }

    // 失败cases详情
    failedCount := 0
    for _, res := range r.Results {
        if !res.Passed {
            failedCount++
        }
    }

    if failedCount > 0 {
        b.WriteString(fmt.Sprintf("### Failed cases (%d)\n\n<details>\n\n", failedCount))
        for _, res := range r.Results {
            if !res.Passed {
                b.WriteString(fmt.Sprintf("**%s** (score=%.2f)\n", res.CaseID, res.Score))
                b.WriteString(fmt.Sprintf("- Input: %s\n", res.Input))
                b.WriteString(fmt.Sprintf("- Expected: %s\n", truncate(res.Expected, 200)))
                b.WriteString(fmt.Sprintf("- Actual: %s\n", truncate(res.Actual, 200)))
                b.WriteString(fmt.Sprintf("- Reason: %s\n\n", res.Reason))
            }
        }
        b.WriteString("</details>\n")
    }

    return b.String()
}

func truncate(s string, n int) string {
    if len(s) <= n { return s }
    return s[:n] + "..."
}

// 1. temperature=0
llm.WithTemperature(0)

// 2. 确定性prompt（输出JSON固定schema）
// 3. 多次跑取中位数
// 4. Judge用强模型（gpt-4o），被测可以弱（gpt-4o-mini）
// 5. 对比模式：只判断"A优于B还是B优于A"，比绝对打分稳定

package main

import (
    "fmt"
    "strconv"
    "strings"
)

type TreeNode struct {
    Val   int
    Left  *TreeNode
    Right *TreeNode
}

// Codec 持有序列化/反序列化方法
type Codec struct{}

// serialize 使用 BFS 层序遍历将树转为字符串
func (c *Codec) serialize(root *TreeNode) string {
    if root == nil {
        return ""
    }
    var sb strings.Builder
    queue := []*TreeNode{root}
    for len(queue) > 0 {
        node := queue[0]
        queue = queue[1:]
        if node == nil {
            sb.WriteString("#,")
        } else {
            sb.WriteString(strconv.Itoa(node.Val))
            sb.WriteByte(',')
            queue = append(queue, node.Left, node.Right)
        }
    }
    return sb.String()
}

// deserialize 从 BFS 序列化字符串还原树
func (c *Codec) deserialize(data string) *TreeNode {
    if data == "" {
        return nil
    }
    tokens := strings.Split(strings.TrimRight(data, ","), ",")
    if len(tokens) == 0 || tokens[0] == "#" {
        return nil
    }
    rootVal, _ := strconv.Atoi(tokens[0])
    root := &TreeNode{Val: rootVal}
    queue := []*TreeNode{root}
    i := 1
    for len(queue) > 0 && i < len(tokens) {
        node := queue[0]
        queue = queue[1:]
        // 处理左子节点
        if i < len(tokens) && tokens[i] != "#" {
            val, _ := strconv.Atoi(tokens[i])
            node.Left = &TreeNode{Val: val}
            queue = append(queue, node.Left)
        }
        i++
        // 处理右子节点
        if i < len(tokens) && tokens[i] != "#" {
            val, _ := strconv.Atoi(tokens[i])
            node.Right = &TreeNode{Val: val}
            queue = append(queue, node.Right)
        }
        i++
    }
    return root
}

func main() {
    c := Codec{}
    root := &TreeNode{
        Val:  1,
        Left: &TreeNode{Val: 2},
        Right: &TreeNode{
            Val:   3,
            Left:  &TreeNode{Val: 4},
            Right: &TreeNode{Val: 5},
        },
    }
    data := c.serialize(root)
    fmt.Println("Serialized:", data)
    restored := c.deserialize(data)
    fmt.Println("Root val:", restored.Val)           // 1
    fmt.Println("Right.Left val:", restored.Right.Left.Val) // 4
}

package main

import (
    "fmt"
    "math"
)

type TreeNode struct {
    Val   int
    Left  *TreeNode
    Right *TreeNode
}

func maxPathSum(root *TreeNode) int {
    maxSum := math.MinInt32

    var dfs func(node *TreeNode) int
    dfs = func(node *TreeNode) int {
        if node == nil {
            return 0
        }
        // 左右子树的最大单向贡献值，负数不选（取0）
        leftGain := dfs(node.Left)
        if leftGain < 0 {
            leftGain = 0
        }
        rightGain := dfs(node.Right)
        if rightGain < 0 {
            rightGain = 0
        }

        // 以当前节点为弯折顶点的最大路径和
        pathThrough := node.Val + leftGain + rightGain
        if pathThrough > maxSum {
            maxSum = pathThrough
        }

        // 返回单向最大值（只能选左或右一侧继续延伸）
        if leftGain > rightGain {
            return node.Val + leftGain
        }
        return node.Val + rightGain
    }

    dfs(root)
    return maxSum
}

func main() {
    // 树：[-10, 9, 20, nil, nil, 15, 7]
    root := &TreeNode{
        Val:  -10,
        Left: &TreeNode{Val: 9},
        Right: &TreeNode{
            Val:   20,
            Left:  &TreeNode{Val: 15},
            Right: &TreeNode{Val: 7},
        },
    }
    fmt.Println(maxPathSum(root)) // 42 (15->20->7)

    // 单节点负数
    fmt.Println(maxPathSum(&TreeNode{Val: -3})) // -3
}

package main

import "fmt"

type TreeNode struct {
    Val   int
    Left  *TreeNode
    Right *TreeNode
}

func buildTree(preorder []int, inorder []int) *TreeNode {
    // 建立中序值→下标的哈希索引
    inorderIdx := make(map[int]int, len(inorder))
    for i, v := range inorder {
        inorderIdx[v] = i
    }

    var build func(preL, preR, inL, inR int) *TreeNode
    build = func(preL, preR, inL, inR int) *TreeNode {
        if preL > preR {
            return nil
        }
        // 前序第一个元素是当前子树的根
        rootVal := preorder[preL]
        root := &TreeNode{Val: rootVal}

        // 在中序中找根的位置
        mid := inorderIdx[rootVal]
        leftSize := mid - inL // 左子树节点数

        // 递归构造左子树和右子树
        root.Left = build(preL+1, preL+leftSize, inL, mid-1)
        root.Right = build(preL+leftSize+1, preR, mid+1, inR)
        return root
    }

    n := len(preorder)
    return build(0, n-1, 0, n-1)
}

func main() {
    preorder := []int{3, 9, 20, 15, 7}
    inorder := []int{9, 3, 15, 20, 7}
    root := buildTree(preorder, inorder)
    fmt.Println(root.Val)        // 3
    fmt.Println(root.Left.Val)   // 9
    fmt.Println(root.Right.Val)  // 20
    fmt.Println(root.Right.Left.Val)  // 15
    fmt.Println(root.Right.Right.Val) // 7
}