package main import ( "context" "strings" "testing" "git.flytoex.net/yuanwei/flyto-agent/pkg/flyto" ) // noopProvider is a minimal ModelProvider for testing buildModelCapabilities // without real API calls. Models() returns nil so base specs fill SourceUntested. type noopProvider struct{} func (n *noopProvider) Name() string { return "noop" } func (n *noopProvider) Stream(_ context.Context, _ *flyto.Request) (<-chan flyto.Event, error) { ch := make(chan flyto.Event) close(ch) return ch, nil } func (n *noopProvider) Models(_ context.Context) ([]flyto.ModelInfo, error) { return nil, nil } // --- lookupDocumented --- func TestLookupDocumented_Hit(t *testing.T) { doc := lookupDocumented("anthropic", "claude-opus-4-6") if doc.Vision == nil || !*doc.Vision { t.Error("expected Vision=true for anthropic:claude-opus-4-6") } if doc.Note == "" { t.Error("expected non-empty Note") } } func TestLookupDocumented_Miss(t *testing.T) { doc := lookupDocumented("nonexistent", "model") if doc.Vision != nil || doc.PDF != nil || doc.MaxTools != nil { t.Error("expected all nil for nonexistent provider:model") } } func TestLookupDocumented_MaxTools(t *testing.T) { doc := lookupDocumented("openrouter", "openai/gpt-4o") if doc.MaxTools == nil { t.Fatal("expected MaxTools for openrouter:openai/gpt-4o") } if *doc.MaxTools != 128 { t.Errorf("expected MaxTools=128, got %d", *doc.MaxTools) } } func TestLookupDocumented_NoMaxTools(t *testing.T) { // Anthropic has no documented MaxTools (normal mode has no hard limit). doc := lookupDocumented("anthropic", "claude-sonnet-4-6") if doc.MaxTools != nil { t.Errorf("expected nil MaxTools for anthropic, got %d", *doc.MaxTools) } } // --- boolCap --- func TestBoolCap_Nil(t *testing.T) { c := boolCap(nil, "test") if c.Source != SourceUntested { t.Errorf("expected SourceUntested, got %s", c.Source) } } func TestBoolCap_True(t *testing.T) { c := boolCap(boolPtr(true), "note") if c.Source != SourceDocumented { t.Errorf("expected SourceDocumented, got %s", c.Source) } if v, ok := c.Value.(bool); !ok || !v { t.Errorf("expected true, got %v", c.Value) } } func TestBoolCap_False(t *testing.T) { c := boolCap(boolPtr(false), "note") if v, ok := c.Value.(bool); !ok || v { t.Errorf("expected false, got %v", c.Value) } } // --- intPtr / boolPtr --- func TestIntPtr(t *testing.T) { p := intPtr(42) if *p != 42 { t.Errorf("expected 42, got %d", *p) } } func TestBoolPtr(t *testing.T) { if !*boolPtr(true) { t.Error("expected true") } if *boolPtr(false) { t.Error("expected false") } } // --- capToTristate --- func TestCapToTristate(t *testing.T) { tests := []struct { name string cap Capability want tristate }{ {"non_probed_source", Capability{Source: SourceDocumented, Value: true}, tsUnknown}, {"bool_true", Capability{Source: SourceProbed, Value: true}, tsYes}, {"bool_false", Capability{Source: SourceProbed, Value: false}, tsNo}, {"float64_positive", Capability{Source: SourceProbed, Value: float64(128)}, tsYes}, {"float64_zero", Capability{Source: SourceProbed, Value: float64(0)}, tsNo}, {"nil_value", Capability{Source: SourceProbed}, tsUnknown}, {"string_value", Capability{Source: SourceProbed, Value: "hello"}, tsUnknown}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := capToTristate(tt.cap); got != tt.want { t.Errorf("got %v, want %v", got, tt.want) } }) } } // --- capToInt --- func TestCapToInt(t *testing.T) { tests := []struct { name string cap Capability want int }{ {"int", Capability{Value: 128}, 128}, {"float64", Capability{Value: float64(64)}, 64}, {"nil", Capability{}, 0}, {"string", Capability{Value: "nope"}, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := capToInt(tt.cap); got != tt.want { t.Errorf("got %d, want %d", got, tt.want) } }) } } // --- capToExhaustive --- func TestCapToExhaustive(t *testing.T) { tests := []struct { name string cap Capability want bool }{ {"nil", Capability{}, false}, {"true", Capability{Exhaustive: boolPtr(true)}, true}, {"false", Capability{Exhaustive: boolPtr(false)}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := capToExhaustive(tt.cap); got != tt.want { t.Errorf("got %v, want %v", got, tt.want) } }) } } // --- isFullyProbed --- func TestIsFullyProbed(t *testing.T) { probed := Capability{Source: SourceProbed} untested := Capability{Source: SourceUntested} tests := []struct { name string mc *ModelCapabilities want bool }{ {"nil", nil, false}, {"all_probed", &ModelCapabilities{ Streaming: probed, Thinking: probed, ToolUse: probed, StructuredOut: probed, Caching: probed, SchemaRef: probed, MaxTools: probed, }, true}, {"MaxTools_untested", &ModelCapabilities{ Streaming: probed, Thinking: probed, ToolUse: probed, StructuredOut: probed, Caching: probed, SchemaRef: probed, MaxTools: untested, }, false}, {"all_empty", &ModelCapabilities{}, false}, {"six_of_seven", &ModelCapabilities{ Streaming: probed, Thinking: probed, ToolUse: probed, StructuredOut: probed, Caching: probed, SchemaRef: untested, MaxTools: probed, }, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := isFullyProbed(tt.mc); got != tt.want { t.Errorf("got %v, want %v", got, tt.want) } }) } } // --- buildModelCapabilities: MaxTools cross-validation (L1175) --- func TestBuildModelCapabilities_MaxToolsCrossValidation(t *testing.T) { ctx := context.Background() noop := &noopProvider{} // Helper: build a CapabilityResult with all probes "succeeded" and specific ToolCount. makeResult := func(provider, model string, toolCount int, exhaustive bool) CapabilityResult { return CapabilityResult{ Provider: provider, Model: model, Streaming: tsYes, Thinking: tsYes, ToolUse: tsYes, StructuredOut: tsYes, Caching: tsYes, SchemaRef: tsYes, ToolCount: toolCount, ToolCountExhaustive: exhaustive, ToolCountNote: "test", } } t.Run("match_non_exhaustive_no_error", func(t *testing.T) { // documented=128, probed=128, exhaustive=false -> no mismatch (128 is lower bound) tgt := target{providerName: "openrouter", model: "openai/gpt-4o", provider: noop} mc := buildModelCapabilities(ctx, tgt, makeResult("openrouter", "openai/gpt-4o", 128, false)) if mc.MaxTools.Evidence == nil { t.Fatal("expected Evidence to be populated") } if mc.MaxTools.Evidence["documented"] != 128 { t.Errorf("expected Evidence[documented]=128, got %v", mc.MaxTools.Evidence["documented"]) } for _, e := range mc.ProbeErrors { if strings.Contains(e, "MaxTools mismatch") { t.Errorf("unexpected mismatch error: %s", e) } } }) t.Run("mismatch_exhaustive_triggers_error", func(t *testing.T) { // documented=128, probed=64, exhaustive=true -> definite mismatch tgt := target{providerName: "openrouter", model: "openai/gpt-4o", provider: noop} mc := buildModelCapabilities(ctx, tgt, makeResult("openrouter", "openai/gpt-4o", 64, true)) found := false for _, e := range mc.ProbeErrors { if strings.Contains(e, "MaxTools mismatch") && strings.Contains(e, "probed=64") && strings.Contains(e, "documented=128") { found = true } } if !found { t.Errorf("expected MaxTools mismatch error, got errors: %v", mc.ProbeErrors) } }) t.Run("match_exhaustive_no_error", func(t *testing.T) { // documented=128, probed=128, exhaustive=true -> exact match, no error tgt := target{providerName: "openrouter", model: "openai/gpt-4o", provider: noop} mc := buildModelCapabilities(ctx, tgt, makeResult("openrouter", "openai/gpt-4o", 128, true)) for _, e := range mc.ProbeErrors { if strings.Contains(e, "MaxTools mismatch") { t.Errorf("unexpected mismatch error for exact match: %s", e) } } }) t.Run("no_documented_no_evidence", func(t *testing.T) { // MiniMax has no documented MaxTools -> no cross-validation tgt := target{providerName: "minimax", model: "MiniMax-M2.7", provider: noop} mc := buildModelCapabilities(ctx, tgt, makeResult("minimax", "MiniMax-M2.7", 128, false)) if mc.MaxTools.Evidence != nil { if _, has := mc.MaxTools.Evidence["documented"]; has { t.Error("expected no documented evidence for MiniMax") } } }) t.Run("zero_tool_count_skips_validation", func(t *testing.T) { // ToolCount=0 -> MaxTools.Source=untested, cross-validation skipped tgt := target{providerName: "openrouter", model: "openai/gpt-4o", provider: noop} result := makeResult("openrouter", "openai/gpt-4o", 0, true) result.ToolCount = 0 mc := buildModelCapabilities(ctx, tgt, result) // Source should be untested (0 means probe failed) if mc.MaxTools.Source != SourceUntested { t.Errorf("expected SourceUntested for ToolCount=0, got %s", mc.MaxTools.Source) } }) }