package capability import ( "context" "encoding/json" "fmt" "sort" "strings" "time" api "git.flytoex.net/yuanwei/flyto-agent/core/internal/transport" "git.flytoex.net/yuanwei/flyto-agent/core/pkg/flyto" ) // defaultMaxProbeTools 是 probeToolCount 二分搜索上界的默认值. // // 超过此数量的工具场景极罕见, 不值得为此花更多 API 调用. 调用方可经 // ProbeOpts.MaxProbeTools 覆盖 (替换原 cmd 写死的 128 常量). // // defaultMaxProbeTools is the default upper bound for the // probeToolCount binary search. Tool counts beyond this are rare and // not worth extra API calls; callers override via // ProbeOpts.MaxProbeTools (replacing the old hardcoded 128 in cmd). const defaultMaxProbeTools = 128 // target 是一个探测目标. type target struct { providerName string provider flyto.ModelProvider model string // thinkingProvider 是专门配置了 ThinkingBudget 的 provider 实例(thinking 探测用). // 目前 thinking 在构造时通过 Config 配置,所以需要单独实例. thinkingProvider flyto.ModelProvider // cachingClient 是直接的 api.Client,用于能正确标记 cache_control 的 Anthropic 兼容端点. // nil = 使用通用 flyto.ModelProvider 路径(无法标记 cache_control) // MiniMax Anthropic 兼容端点需要此字段才能探测 caching. cachingClient *api.Client // cachingProvider 是配置了 EnableCaching=true 的 provider 实例(OpenRouter caching 探测用). // 精妙之处(CLEVER): OpenRouter caching 探测不能复用 cachingClient(那是 api.Client 直连 Anthropic), // 也不能复用 provider(未开启 EnableCaching,系统消息格式不含 cache_control). // 单独实例隔离配置,不影响其他能力探测的正常请求. // nil = 跳过 OpenRouter 路径,直接用 probeCachingGeneric(无法主动建立缓存). cachingProvider flyto.ModelProvider // ADR-0007 § 2.2 ProviderKind 静态标: "direct" (anthropic / openai / // minimax / gemini / deepseek 第一方端点) vs "aggregator" (openrouter // 等聚合层). 注册期人类知识填充, 不实测 — probe 不试图区分这层. // // ADR-0007 § 2.2 ProviderKind static label: "direct" (anthropic / // openai / minimax / gemini / deepseek first-party endpoints) vs // "aggregator" (openrouter etc). Filled at registration time from // human knowledge; never probed. providerKind string // --- 成本控制 (ADR-0018): 贵探针的 skip 开关 + 可配置 MaxProbeTools --- // // 这些字段从 ProbeOpts 透传进来, 让 probe() 跳过昂贵探针 (caching 阶梯 // ~14400 token / maxOutputTokens 128k 预算 / toolCount 二分 ~9 次调用 / // reasoningPassback 2 往返 / schemaFeatures 4 子探针). skip 时对应能力 // 留在零值 Capability (即 SourceUntested), 不实测. // // Cost controls (ADR-0018): skip flags for the expensive probes plus a // configurable MaxProbeTools, threaded in from ProbeOpts so probe() // can skip the costly probes (caching ladder ~14400 tokens / // maxOutputTokens 128k budget / toolCount binary search ~9 calls / // reasoningPassback 2 round-trips / schemaFeatures 4 sub-probes). When // skipped, the capability is left as its zero Capability (i.e. // SourceUntested), not probed. maxProbeTools int skipCaching bool skipMaxOutputTokens bool skipToolCount bool skipReasoningPassback bool skipSchemaFeatures bool } // ProbeOpts 携带 probe 需要的四个 provider 句柄 + 静态 providerKind 标签 // + per-probe skip 控制. // // 升华改进(ELEVATED): 单 provider 签名不足 — caching 与 thinking 需各自 // 单独配置的实例 (ThinkingBudget / EnableCaching / cache_control 直路). // 把四个句柄折进 ProbeOpts, 既保留 cmd 原有的 key-dependent 构造灵活性, // 又给 server 等消费者一个干净的公共入口. // // ProbeOpts carries the four provider-ish handles the probes need plus // the static providerKind label and per-probe skip controls. A single // provider signature is insufficient -- caching and thinking each need // separately configured instances (ThinkingBudget / EnableCaching / // the cache_control direct path). Folding the four handles into // ProbeOpts keeps cmd's key-dependent construction flexibility while // giving server-side consumers a clean public entry. type ProbeOpts struct { // ProviderName 是结果标注用的 provider 名 (如 "anthropic"). // ProviderName is the provider label for results (e.g. "anthropic"). ProviderName string // Provider 是必需的基础 provider 句柄. // Provider is the REQUIRED base provider handle. Provider flyto.ModelProvider // ThinkingProvider 是配置了 ThinkingBudget 的实例; nil -> 跳过 thinking. // ThinkingProvider is a ThinkingBudget-configured instance; nil -> skip thinking. ThinkingProvider flyto.ModelProvider // CachingClient 是直连 internal/transport 的 client (Anthropic / MiniMax // cache_control 直路); nil -> 跳过该路径. // CachingClient is a direct internal/transport client for the // Anthropic / MiniMax cache_control path; nil -> skip that path. CachingClient *api.Client // CachingProvider 是 EnableCaching=true 实例 (OpenRouter) 或同一实例 // (DeepSeek long-prefix 阶梯); nil -> 走 generic 路径. // CachingProvider is an EnableCaching=true instance (OpenRouter) or the // same instance (DeepSeek long-prefix ladder); nil -> generic path. CachingProvider flyto.ModelProvider // ProviderKind 是静态 "direct" / "aggregator" 标签. // ProviderKind is the static "direct" / "aggregator" label. ProviderKind string // --- 成本控制: 贵探针可跳过 (见 target 注释) --- // Cost controls: the expensive probes can be skipped (see target docs). MaxProbeTools int // 0 -> 默认 128 (替换原写死值). 0 -> default 128 (replaces hardcode). SkipCaching bool SkipMaxOutputTokens bool SkipToolCount bool SkipReasoningPassback bool SkipSchemaFeatures bool } // ProbeModel 跑完整 probe + merge 流水线, 返回一个 model 的 // probed + Models() + documented 合并能力画像. // // 它从 opts 构造内部 target, 运行 probe() + buildModelCapabilities, // 返回 *ModelCapabilities. probe() 本身从不 hard-fail (诊断记进 Note // 后继续), 故 error 仅保留给 nil-Provider 校验 / ctx 取消. // // ProbeModel runs the full probe + merge pipeline for one model and // returns the merged probed + Models() + documented capability picture. // It builds the internal target from opts, runs probe() + // buildModelCapabilities, and returns *ModelCapabilities. probe() // itself never hard-fails (it records diagnostics in Notes and // continues), so error is reserved for nil-Provider validation / ctx // cancellation. func ProbeModel(ctx context.Context, model string, opts ProbeOpts) (*ModelCapabilities, error) { if opts.Provider == nil { return nil, fmt.Errorf("capability: ProbeModel requires a non-nil Provider") } t := targetFromOpts(model, opts) r := probe(ctx, t) return buildModelCapabilities(ctx, t, r), nil } // Probe 是想要原始 tristate 结果的低层入口 (包装 probe()). // // Probe is a lower-level entry for callers wanting the raw tristate // result (wraps probe()). func Probe(ctx context.Context, opts ProbeOpts, model string) CapabilityResult { return probe(ctx, targetFromOpts(model, opts)) } // IsFullyProbed 是 isFullyProbed 的公共导出版 (缓存 / skip 消费者用的完整性检查). // // IsFullyProbed is the exported wrapper of isFullyProbed (a // completeness check for cache / skip consumers). func IsFullyProbed(mc *ModelCapabilities) bool { return isFullyProbed(mc) } // targetFromOpts 把 ProbeOpts 折成内部 target, 应用 MaxProbeTools 默认值. // // targetFromOpts folds ProbeOpts into the internal target, applying the // MaxProbeTools default. func targetFromOpts(model string, opts ProbeOpts) target { maxTools := opts.MaxProbeTools if maxTools <= 0 { maxTools = defaultMaxProbeTools } return target{ providerName: opts.ProviderName, provider: opts.Provider, model: model, thinkingProvider: opts.ThinkingProvider, cachingClient: opts.CachingClient, cachingProvider: opts.CachingProvider, providerKind: opts.ProviderKind, maxProbeTools: maxTools, skipCaching: opts.SkipCaching, skipMaxOutputTokens: opts.SkipMaxOutputTokens, skipToolCount: opts.SkipToolCount, skipReasoningPassback: opts.SkipReasoningPassback, skipSchemaFeatures: opts.SkipSchemaFeatures, } } // isFullyProbed 检查一个 ModelCapabilities 的 7 个可实测字段是否全部 Source==probed. // // 精妙之处(CLEVER): 只检查可实测字段 (Streaming/Thinking/ToolUse/StructuredOut/ // Caching/SchemaRef/MaxTools). 文档字段 (Vision/PDF/Batch 等) 和基础规格 // (ContextWindow/Price 等) 不影响判断--它们来自静态表,每次都会重新填充. // 任何一个非 probed (包括 untested / empty) 都触发重新探测整个 target, // 因为 7 个探测按顺序执行(streaming 失败会 skip 后续),单独重跑某一个不可靠. func isFullyProbed(mc *ModelCapabilities) bool { if mc == nil { return false } fields := []Source{ mc.Streaming.Source, mc.Thinking.Source, mc.ToolUse.Source, mc.StructuredOut.Source, mc.Caching.Source, mc.SchemaRef.Source, mc.MaxTools.Source, } for _, s := range fields { if s != SourceProbed { return false } } return true } // buildModelCapabilities 将一次 probe 结果 + provider.Models() 元数据 + documented 表 // 合并为一个完整的 ModelCapabilities 对象. // // 精妙之处(CLEVER): 数据来源三分层-- // 1. 可实测字段(Streaming/Thinking/ToolUse/StructuredOut/Caching/SchemaRef/MaxTools) // → SourceProbed,Value 取 tristate 对应 bool,Evidence 记诊断信息 // 2. 基础规格(Context/Tokens/Price)从 provider.Models() 查,SourceDocumented // 如果 provider 返回 error 或未命中此 model,标 SourceUntested // 3. 文档字段(Vision/PDF/Batch/Parallel/Strict)从 documentedCapabilities 表查, // 未命中的标 SourceUntested // // 反向思维:是否应该把 Models() 查找移到 main 循环之外统一做一次? // 否决--provider 的 Models() 是本地静态表查找,代价可忽略, // 每次 target 单独查反而让 buildModelCapabilities 自包含无副作用,更易测试. func buildModelCapabilities(ctx context.Context, t target, r CapabilityResult) *ModelCapabilities { mc := &ModelCapabilities{ Provider: t.providerName, Model: t.model, ProbedAt: time.Now().UTC().Format(time.RFC3339), } // --- 1. 可实测字段(SourceProbed)--- mc.Streaming = tristateToCapability(r.Streaming, extractNote(r.Notes, "stream:")) mc.Thinking = tristateToCapability(r.Thinking, extractNote(r.Notes, "think:")) mc.ToolUse = tristateToCapability(r.ToolUse, extractNote(r.Notes, "tool:")) mc.StructuredOut = tristateToCapability(r.StructuredOut, extractNote(r.Notes, "struct:")) mc.Caching = tristateToCapability(r.Caching, extractNote(r.Notes, "cache:")) mc.SchemaRef = tristateToCapability(r.SchemaRef, extractNote(r.Notes, "schemaref:")) if r.ToolCount > 0 { exh := r.ToolCountExhaustive mc.MaxTools = Capability{ Value: r.ToolCount, Source: SourceProbed, Exhaustive: &exh, Note: r.ToolCountNote, Evidence: map[string]any{ "max_tested": r.ToolCount, // 二分搜索结束位置:穷尽时是 lo+1(首次拒绝点),未穷尽时为 null "first_refusal_at": func() any { if r.ToolCountExhaustive && r.ToolCount > 0 && r.ToolCount < 128 { return r.ToolCount + 1 } return nil }(), }, } } else { // 0 表示连 1 个工具都被拒绝;这本身是穷尽结果(确定性 false). exh := true mc.MaxTools = Capability{ Source: SourceUntested, // 注意:依然标 Untested,因为没有 Value 可供下游使用 Exhaustive: &exh, Note: r.ToolCountNote, } } // --- 2. 基础规格(SourceDocumented,从 provider.Models() 查)--- var matched *flyto.ModelInfo if models, err := t.provider.Models(ctx); err == nil { for i := range models { if models[i].ID == t.model { matched = &models[i] break } } } if matched != nil { mc.ContextWindow = Capability{Value: matched.ContextWindow, Source: SourceDocumented} mc.MaxOutputTokens = Capability{Value: matched.MaxOutputTokens, Source: SourceDocumented} mc.InputPricePer1M = Capability{Value: matched.InputPricePer1M, Source: SourceDocumented} mc.OutputPricePer1M = Capability{Value: matched.OutputPricePer1M, Source: SourceDocumented} if matched.CacheReadPricePer1M > 0 { mc.CacheReadPricePer1M = Capability{Value: matched.CacheReadPricePer1M, Source: SourceDocumented} } else { mc.CacheReadPricePer1M = Capability{Source: SourceUntested} } if matched.CacheWritePricePer1M > 0 { mc.CacheWritePricePer1M = Capability{Value: matched.CacheWritePricePer1M, Source: SourceDocumented} } else { mc.CacheWritePricePer1M = Capability{Source: SourceUntested} } } else { // provider.Models() 未命中此 model(如 MiniMax-M2.7-highspeed 不在静态表) mc.ContextWindow = Capability{Source: SourceUntested} mc.MaxOutputTokens = Capability{Source: SourceUntested} mc.InputPricePer1M = Capability{Source: SourceUntested} mc.OutputPricePer1M = Capability{Source: SourceUntested} mc.CacheReadPricePer1M = Capability{Source: SourceUntested} mc.CacheWritePricePer1M = Capability{Source: SourceUntested} } // --- 3. 文档字段(从 documentedCapabilities 查)--- doc := lookupDocumented(t.providerName, t.model) mc.Vision = boolCap(doc.Vision, doc.Note) mc.PDF = boolCap(doc.PDF, doc.Note) mc.Batch = boolCap(doc.Batch, doc.Note) mc.ParallelToolCalls = boolCap(doc.ParallelToolCalls, doc.Note) mc.StrictJSON = boolCap(doc.StrictJSON, doc.Note) // --- 4. MaxTools 文档交叉验证 (L1175) --- // // 精妙之处(CLEVER): 只在 exhaustive=true (确认硬上限) 时才报 mismatch-- // exhaustive=false 表示"测到 N 还能过", probed=128 vs documented=128 此时 // 不是矛盾而是"下界刚好等于文档值", 真上限可能更大. // 替代方案: - 否决: 大多数 provider 会走快速路径 // (tryN(128) 直接过), 返回 exhaustive=false, 与 documented=128 不是真冲突. if doc.MaxTools != nil && mc.MaxTools.Source == SourceProbed { documented := *doc.MaxTools probed := capToInt(mc.MaxTools) exhaustive := mc.MaxTools.Exhaustive != nil && *mc.MaxTools.Exhaustive if mc.MaxTools.Evidence == nil { mc.MaxTools.Evidence = map[string]any{} } mc.MaxTools.Evidence["documented"] = documented if documented > 0 && exhaustive && probed != documented { mc.ProbeErrors = append(mc.ProbeErrors, fmt.Sprintf( "MaxTools mismatch: probed=%d (exhaustive) documented=%d", probed, documented)) } } // --- 5. SchemaFeatures 转 Capability (L1174) --- if len(r.SchemaFeatures) > 0 { mc.SchemaFeatures = make(map[string]Capability, len(r.SchemaFeatures)) for name, supported := range r.SchemaFeatures { mc.SchemaFeatures[name] = Capability{ Value: supported, Source: SourceProbed, } } } // --- 5.5. ADR-0007 capability tracking 三件套 (TD-20) --- // ProviderKind 是注册时静态标 (SourceManual = 人工标注的专家知识). // ToolNameRegex / ReasoningPassbackMode 实测填 SourceProbed; 空值 = 未达 // 探测前置条件 (Tool/Thinking 必须 ✓), 标 SourceUntested. // // ADR-0007 capability tracking triplet (TD-20). // ProviderKind is registration-time static (SourceManual = expert // knowledge). ToolNameRegex / ReasoningPassbackMode are probed when // preconditions met (SourceProbed); empty values from missed // preconditions become SourceUntested. if r.ProviderKind != "" { mc.ProviderKind = Capability{Value: r.ProviderKind, Source: SourceManual, Note: "registration-time label"} } else { mc.ProviderKind = Capability{Source: SourceUntested} } if r.ToolNameRegex != "" { mc.ToolNameRegex = Capability{Value: r.ToolNameRegex, Source: SourceProbed, Note: r.ToolNameRegexNote} } else { mc.ToolNameRegex = Capability{Source: SourceUntested, Note: r.ToolNameRegexNote} } if r.ReasoningPassbackMode != "" { mc.ReasoningPassbackMode = Capability{Value: r.ReasoningPassbackMode, Source: SourceProbed, Note: r.ReasoningPassbackNote} } else { mc.ReasoningPassbackMode = Capability{Source: SourceUntested, Note: r.ReasoningPassbackNote} } // --- 6. probe 错误收集 --- for _, note := range r.Notes { if strings.Contains(note, "err:") || strings.Contains(note, "ERR") { mc.ProbeErrors = append(mc.ProbeErrors, note) } } return mc } // tristateToCapability 将 tristate 转换为带来源的 Capability. // // 精妙之处(CLEVER): tsUnknown 映射到 SourceUntested 而非 SourceProbed-- // unknown 意味着 probe 未执行(如 streaming 失败后跳过), // 这时候说"实测不支持"会误导,应标"未测". func tristateToCapability(t tristate, note string) Capability { switch t { case tsYes: return Capability{Value: true, Source: SourceProbed, Note: note} case tsNo: return Capability{Value: false, Source: SourceProbed, Note: note} case tsError: return Capability{Source: SourceProbed, Note: "probe error: " + note} default: return Capability{Source: SourceUntested, Note: note} } } // extractNote 从 Notes 列表中按前缀提取单条 note(去掉前缀). func extractNote(notes []string, prefix string) string { for _, n := range notes { if strings.HasPrefix(n, prefix) { return strings.TrimPrefix(n, prefix) } } return "" } // probe 执行所有能力探测,返回结果(含诊断信息). // // 升华改进(ELEVATED): 早期方案每个探测只返回 tristate,✗ 原因不明-- // 不知道是"真不支持"还是"测法错了". // 新版每个探测额外返回诊断字符串:错误信息,实际输出,token 数等, // 所有诊断进入 Notes 列,可直接从矩阵读出失败原因. // caching 探测额外做自适应探测(见 probeCachingAnthropic/probeCachingProvider). // // 门控契约 (ADR-0018 逐字保留): streaming 门控一切 (失败立即返回); // schema_features / tool_name_regex 需 ToolUse==yes; reasoning_passback // 需 ToolUse==yes AND Thinking==yes; providerKind 是静态标 (无 API 调用). // 此外 target 的 skip* 开关让贵探针 (caching / maxOutputTokens / toolCount // / reasoningPassback / schemaFeatures) 在置位时跳过, 对应能力留零值. // // Gating contract (ADR-0018, preserved verbatim): streaming gates // everything (early-return on failure); schema_features / // tool_name_regex need ToolUse==yes; reasoning_passback needs // ToolUse==yes AND Thinking==yes; providerKind is a static label (no // API call). Additionally the target's skip* flags cause the expensive // probes (caching / maxOutputTokens / toolCount / reasoningPassback / // schemaFeatures) to be skipped when set, leaving the capability zero. func probe(ctx context.Context, t target) CapabilityResult { r := CapabilityResult{ Provider: t.providerName, Model: t.model, } var diag string // 1. Streaming(基础连通性) r.Streaming, diag, _ = probeStreaming(ctx, t.provider, t.model) if diag != "" { r.Notes = append(r.Notes, "stream:"+diag) } // 后续探测只有 streaming 通了才有意义 if r.Streaming != tsYes { r.Notes = append(r.Notes, "streaming 失败,跳过后续") return r } // 2. Thinking r.Thinking, diag, _ = probeThinking(ctx, t.thinkingProvider, t.model) if diag != "" { r.Notes = append(r.Notes, "think:"+diag) } // 3. Tool Use r.ToolUse, diag, _ = probeToolUse(ctx, t.provider, t.model) if diag != "" { r.Notes = append(r.Notes, "tool:"+diag) } // 4. Structured Output r.StructuredOut, diag, _ = probeStructuredOutput(ctx, t.provider, t.model) if diag != "" { r.Notes = append(r.Notes, "struct:"+diag) } // 5. Caching - 无论结果如何都记录诊断(原始 token 数是最重要的证据) // skipCaching 置位时跳过 (贵探针: caching 阶梯 ~14400 token), Caching 留零值. // skipCaching skips the expensive caching ladder (~14400 tokens), leaving zero. if !t.skipCaching { r.Caching, diag, _ = probeCaching(ctx, t) if diag != "" { r.Notes = append(r.Notes, "cache:"+diag) } } // 6. SchemaRef - 工具 InputSchema 中的 $ref 是否被正确解析 r.SchemaRef, diag, _ = probeSchemaRef(ctx, t.provider, t.model) if diag != "" { r.Notes = append(r.Notes, "schemaref:"+diag) } // 7. ToolCount - 模型可处理的最大工具数量(二分探测) // skipToolCount 置位时跳过 (贵探针: 二分 ~9 次调用, 每次最多 128 tool defs). // skipToolCount skips the expensive binary search (~9 calls, up to 128 tool defs each). if !t.skipToolCount { r.ToolCount, r.ToolCountExhaustive, r.ToolCountNote, _ = probeToolCount(ctx, t.provider, t.model, t.maxProbeTools) if r.ToolCountNote != "" { r.Notes = append(r.Notes, "toolcount:"+r.ToolCountNote) } } // 7.5 MaxOutputTokens - 单次响应输出 token 实测上限 // 修 ADR-0005 Bug S follow-up: ModelRegistry.MaxOutputTokens 是 model // capability 字段, day-1 起 flyto.ModelInfo 就有但无 probe 工具实测 — 各 // provider 静态表里填的多是文档值或推断值. 加此 probe 让 caller 跑一次 // 拿真值 + 更新静态表. // // Fix ADR-0005 Bug S follow-up: ModelRegistry.MaxOutputTokens is a model // capability field that has been in flyto.ModelInfo since day-1 but had // no probe tool — provider static tables filled values from docs or // inference. This probe lets callers run once for ground-truth values // and refresh static tables. // // skipMaxOutputTokens 置位时跳过 (贵探针: 128k 预算, 索要 50k+ 输出 token). // skipMaxOutputTokens skips the expensive probe (128k budget, demands 50k+ output tokens). if !t.skipMaxOutputTokens { r.MaxOutputTokens, r.MaxOutputTokensExhaustive, r.MaxOutputTokensNote, _ = probeMaxOutputTokens(ctx, t.provider, t.model) if r.MaxOutputTokensNote != "" { r.Notes = append(r.Notes, "maxout:"+r.MaxOutputTokensNote) } } // 8. SchemaFeatures - JSON Schema 特性支持 (L1174) // 前置条件: ToolUse 必须通过, 否则发包含工具的请求无意义. // skipSchemaFeatures 置位时跳过 (4 子探针). // skipSchemaFeatures skips the 4 sub-probes. if !t.skipSchemaFeatures && r.ToolUse == tsYes { r.SchemaFeatures = probeSchemaFeatures(ctx, t.provider, t.model) var parts []string for name, ok := range r.SchemaFeatures { if ok { parts = append(parts, name+"=✓") } else { parts = append(parts, name+"=✗") } } if len(parts) > 0 { // 排序保证输出稳定 sort.Strings(parts) r.Notes = append(r.Notes, "schema:"+strings.Join(parts, ",")) } } // 9. ADR-0007 capability tracking 三件套 (TD-20) // // ProviderKind: 静态来自 target 注册标签, 不发请求. // ToolNameRegex: 实测 (前置 ToolUse=tsYes, 否则探不到协议反应). // ReasoningPassbackMode: 实测 (前置 ToolUse=tsYes + Thinking=tsYes, // 否则 round 1 拿不到 reasoning_content + tool_use 双产物). // // ADR-0007 capability tracking triplet (TD-20). // ProviderKind: static from registration label, no API call. // ToolNameRegex: probed (precondition ToolUse=tsYes). // ReasoningPassbackMode: probed (precondition ToolUse=tsYes // AND Thinking=tsYes — round 1 must yield both). r.ProviderKind = t.providerKind if t.providerKind != "" { r.Notes = append(r.Notes, "kind:"+t.providerKind+"(static)") } if r.ToolUse == tsYes { r.ToolNameRegex, r.ToolNameRegexNote, _ = probeToolNameRegex(ctx, t.provider, t.model) if r.ToolNameRegexNote != "" { r.Notes = append(r.Notes, "regex:"+r.ToolNameRegexNote) } } // skipReasoningPassback 置位时跳过 (贵探针: 2 往返, 90s). // skipReasoningPassback skips the expensive probe (2 round-trips, 90s). if !t.skipReasoningPassback && r.ToolUse == tsYes && r.Thinking == tsYes { r.ReasoningPassbackMode, r.ReasoningPassbackNote, _ = probeReasoningPassback(ctx, t.thinkingProvider, t.model) if r.ReasoningPassbackNote != "" { r.Notes = append(r.Notes, "passback:"+r.ReasoningPassbackNote) } } return r } // probeStreaming 发最小请求,验证 SSE 流是否正常返回文本. func probeStreaming(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() ch, err := p.Stream(ctx, &flyto.Request{ Model: model, MaxTokens: 5, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Reply with the word OK only.")}}, }, }) if err != nil { return tsError, fmt.Sprintf("connect err: %v", err), err } for evt := range ch { switch e := evt.(type) { case *flyto.TextDeltaEvent, *flyto.ThinkingDeltaEvent: // 精妙之处(CLEVER): ThinkingDeltaEvent 同样证明 SSE streaming 正常工作. // MiniMax-M2.7 thinking 默认开启,小 token 限制(max_tokens=5)时 // 全部 token 消耗在 reasoning 上,content 为空--若只接受 TextDeltaEvent // 会误报 ✗,实际 streaming 完全正常. drain(ch) return tsYes, "", nil case *flyto.ErrorEvent: return tsError, fmt.Sprintf("err: %v", e.Err), nil } } return tsNo, "no text/thinking events received", nil } // probeThinking 发含 thinking_budget 的请求,检测响应是否含 ThinkingEvent. // 使用带 ThinkingBudget 配置的 provider 实例. func probeThinking(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { ctx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() ch, err := p.Stream(ctx, &flyto.Request{ Model: model, // 精妙之处(CLEVER): MaxTokens 必须 >= ThinkingBudget(当前 1024),否则 Anthropic 返回 400. // 额外留出 1024 给实际回复,避免思考消耗完预算后无法产出文本. MaxTokens: 2048, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("What is 3+5? Think step by step.")}}, }, }) if err != nil { return tsError, fmt.Sprintf("connect err: %v", err), err } for evt := range ch { switch e := evt.(type) { case *flyto.ThinkingEvent, *flyto.ThinkingDeltaEvent: drain(ch) return tsYes, "", nil case *flyto.ErrorEvent: return tsNo, fmt.Sprintf("err: %v", e.Err), nil } } return tsNo, "no thinking events (model may not support or budget too low)", nil } // probeToolUse 发含工具定义的请求,检测模型是否返回 tool_use block. func probeToolUse(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() req := &flyto.Request{ Model: model, // 精妙之处(CLEVER): MaxTokens=256 而非 64-- // Claude 4.x 模型在工具调用前可能有隐式 reasoning overhead, // 64 tokens 不足以完成 thinking + tool_use JSON block. MaxTokens: 256, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("What is the weather in Beijing? Use the get_weather tool.")}}, }, Tools: []flyto.Tool{ { Name: "get_weather", Description: "Get current weather for a city", InputSchema: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), }, }, } ch, err := p.Stream(ctx, req) if err != nil { return tsError, fmt.Sprintf("connect err: %v", err), err } for evt := range ch { switch e := evt.(type) { case *flyto.ToolUseEvent: drain(ch) return tsYes, "", nil case *flyto.ErrorEvent: return tsNo, fmt.Sprintf("err: %v", e.Err), nil } } return tsNo, "no tool_use event (model responded in text only)", nil } // probeStructuredOutput 发含 ResponseFormat=json_object 的请求,检测输出是否为合法 JSON. // // 精妙之处(CLEVER): ResponseFormat 双保险-- // system prompt 引导 + response_format.type=json_object 双重约束. // 只用 system prompt 时,模型有时添加 markdown 代码块或解释文字,导致误判. // json_object 模式由 provider 在协议层强制约束,更可靠. func probeStructuredOutput(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { ctx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() ch, err := p.Stream(ctx, &flyto.Request{ Model: model, // 精妙之处(CLEVER): 256 tokens 而非 50-- // MiniMax-M2.7 thinking 默认开启,thinking 本身消耗约 50-100 tokens, // 50 tokens 的限制会在 JSON 输出中途截断,导致 json.Unmarshal 失败. // 256 tokens 足够覆盖 thinking overhead + 完整 JSON 输出. MaxTokens: 256, System: `Respond ONLY with valid JSON, no markdown fences, no explanations. Format: {"name":"string","score":number}`, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Give me a fake person named Alice with score 95.")}}, }, ResponseFormat: &flyto.ResponseFormat{Type: "json_object"}, }) if err != nil { return tsError, fmt.Sprintf("connect err: %v", err), err } // 精妙之处(CLEVER): TextEvent 优先于 TextDeltaEvent 累积-- // wire 层同时发 TextDeltaEvent(逐块增量)和 TextEvent(finish 时的完整文本). // 若同时累积两者,buf 会变成 "delta1+delta2+...+fullText",即两倍文本,JSON 解析必失败. // 策略:优先取 TextEvent(完整,出现一次),fallback 到 TextDeltaEvent 累积. var deltaBuf strings.Builder var finalText string var streamErr string for evt := range ch { switch e := evt.(type) { case *flyto.TextDeltaEvent: deltaBuf.WriteString(e.Text) case *flyto.TextEvent: finalText = e.Text // 覆盖增量累积,取完整文本 case *flyto.ErrorEvent: streamErr = fmt.Sprintf("err: %v", e.Err) } } if streamErr != "" { return tsError, streamErr, nil } text := finalText if text == "" { text = deltaBuf.String() } text = strings.TrimSpace(text) // 精妙之处(CLEVER): 剥掉 markdown 代码块(```json ... ``` 或 ``` ... ```)-- // Haiku 4.5 即使系统提示明确要求"no markdown fences",仍会包裹代码块: // ```json\n{"name":"Alice","score":95}\n``` // json.Unmarshal 遇到反引号会立即失败,误判为"不支持结构化输出". // 剥掉外层 fence 后再解析,正确反映模型确实输出了合法 JSON. // 替代方案:<在系统提示中更强调禁止 markdown> - 否决: // 实测 prompt engineering 在小模型上不可靠,防御性 strip 更健壮. for _, prefix := range []string{"```json\n", "```\n", "```json", "```"} { if strings.HasPrefix(text, prefix) { text = strings.TrimPrefix(text, prefix) break } } for _, suffix := range []string{"\n```", "```"} { if strings.HasSuffix(text, suffix) { text = strings.TrimSuffix(text, suffix) break } } text = strings.TrimSpace(text) var v map[string]any if err := json.Unmarshal([]byte(text), &v); err != nil { // 诊断:截取实际输出前 80 字符,帮助判断是 format 问题还是模型不支持 preview := text if len(preview) > 80 { preview = preview[:80] + "..." } return tsNo, fmt.Sprintf("json-parse-err, output=%q", preview), nil } return tsYes, "", nil } // probeCaching 检测 provider 是否支持 prompt caching. // // 三条路径: // 1. cachingClient != nil(Anthropic/MiniMax 直连):直接用 api.Client 发 cache_control 请求. // 2. cachingProvider != nil(OpenRouter → Anthropic 路径):用配置了 EnableCaching=true // 的 provider 发自适应长度系统提示,检测 cached_tokens > 0. // 3. 通用路径:发两次相同请求,检测 cache_read_tokens > 0(依赖 provider 自动缓存). func probeCaching(ctx context.Context, t target) (tristate, string, error) { if t.cachingClient != nil { return probeCachingAnthropic(ctx, t.cachingClient, t.model) } if t.cachingProvider != nil { return probeCachingProvider(ctx, t.cachingProvider, t.model) } return probeCachingGeneric(ctx, t.provider, t.model) } // cachingReqClient 发一次 Anthropic caching 请求,返回 (inputTokens, creationTokens, readTokens, err). // // 精妙之处(CLEVER): 拆成独立 helper 是为了让 probeCachingAnthropic 能复用同一个请求逻辑 // 做自适应循环--不同长度的 system 只是参数变化,其余完全相同. // 同时捕获 ErrorEvent,避免错误被静默吞掉(channel 关闭但无 error 返回). // // 升华改进(ELEVATED): 早期方案只返回 (creation, read)--遇到 cr=0 无法判断 // 是"系统提示太短没达到阈值"还是"API 根本没算到 input_tokens". // 新版从 usage.input_tokens 读出实际计费的 input 数,让外层自适应探测 // 根据真实 token 数而非理论估算(reps*12)决定是否加长. func cachingReqClient(ctx context.Context, client *api.Client, model, system string) (inputTokens, creationTokens, readTokens int, err error) { reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() req := &api.MessageRequest{ Model: model, MaxTokens: 5, Beta: &api.BetaFeatures{PromptCaching: true}, } req.SetSystemBlocks([]api.SystemContentBlock{{ Type: "text", Text: system, CacheControl: &api.CacheControl{Type: "ephemeral"}, }}) req.Messages = []api.RequestMessage{api.NewTextMessage("user", "Say hi.")} ch, e := client.CreateMessageStream(reqCtx, req) if e != nil { return 0, 0, 0, e } for evt := range ch { switch ev := evt.(type) { case *flyto.UsageEvent: inputTokens = ev.InputTokens creationTokens = ev.CacheCreationTokens readTokens = ev.CacheReadTokens case *flyto.ErrorEvent: return 0, 0, 0, ev.Err } } return } // cachingReqProvider 发一次 provider caching 请求,返回 (inputTokens, creationTokens, readTokens, err). func cachingReqProvider(ctx context.Context, p flyto.ModelProvider, model, system string) (inputTokens, creationTokens, readTokens int, err error) { reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() ch, e := p.Stream(reqCtx, &flyto.Request{ Model: model, MaxTokens: 5, System: system, Messages: []flyto.Message{{Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Say hi.")}}}, }) if e != nil { return 0, 0, 0, e } for evt := range ch { switch ev := evt.(type) { case *flyto.UsageEvent: inputTokens = ev.InputTokens creationTokens = ev.CacheCreationTokens readTokens = ev.CacheReadTokens case *flyto.ErrorEvent: return 0, 0, 0, ev.Err } } return } // probeCachingAnthropic 自适应探测 Anthropic 兼容端点的 prompt caching. // // 升华改进(ELEVATED): 早期方案硬编码 200 次重复(~2700t),靠猜是否超过阈值-- // 遇到 ✗ 时无法区分"token 不够"和"真不支持". // 新版逐步加长系统提示,通过 API 响应的 cache_creation_input_tokens 精确判断: // // cr=0 → 这个长度不够,继续加长 // cr>0, rd>0 → 缓存建立且命中,✓ // cr>0, rd=0 → 建立了但没命中(请求不一致 bug,理论上不应发生) // 所有长度均 cr=0 → 真的不支持或 tier 限制 // // bug 修复(FIXED): 早期方案有两个 bug-- // 1. 若上一次 probe 已把缓存建好,这次第一次请求会得到 cr=0 rd>0(直接命中), // 原逻辑 cr==0 continue 会错过这种情况,误判为 ✗. // 新版:第一次请求若 cr=0 rd>0,直接返回 tsYes(缓存已存在于 tier). // 2. 实际 token 数只靠 reps*12 估算,没有 ground truth-- // 新版从 usage.input_tokens 读出真实 token 数打印到 diag. // // 梯度(次数 → 约 tokens): // // 100 → ~1200t(覆盖 Sonnet 1024 阈值) // 300 → ~3600t(覆盖 Haiku 2048 阈值,低于 4096) // 600 → ~7200t(覆盖 Haiku 4096 阈值) // 1200 → ~14400t(为未知高阈值模型兜底) func probeCachingAnthropic(ctx context.Context, client *api.Client, model string) (tristate, string, error) { phrase := "You are a helpful AI assistant that answers concisely. " for _, reps := range []int{100, 300, 600, 1200} { system := strings.Repeat(phrase, reps) approxT := reps * 12 in1, cr, rd, err := cachingReqClient(ctx, client, model, system) if err != nil { return tsError, fmt.Sprintf("err@~%dt(in=%d): %v", approxT, in1, err), err } // bug 修复(FIXED): 第一次 cr=0 rd>0 表示缓存已在 tier 中存在(上次 probe 残留), // 直接返回 tsYes--这本身就是 caching 支持的铁证. if cr == 0 && rd > 0 { return tsYes, fmt.Sprintf("cache-hit-pre-existing in=%d cr=0 rd=%d (~%dt)", in1, rd, approxT), nil } if cr == 0 { continue // 未触发缓存建立,加长后重试 } // 缓存已建立,第二次验证命中 // 精妙之处(CLEVER): 同时捕获第二次请求的 cr2 和 rd-- // cr2>0 rd=0 → 每次都重新建缓存(cache key 不一致或 beta header 问题) // cr2=0 rd>0 → 正常命中 // cr2=0 rd=0 → 缓存在两次请求之间过期(TTL 异常) in2, cr2, rd2, err := cachingReqClient(ctx, client, model, system) if err != nil { return tsError, fmt.Sprintf("cr=%d in1=%d err@2nd: %v", cr, in1, err), err } if rd2 > 0 { return tsYes, fmt.Sprintf("in=%d/%d cr=%d rd=%d (~%dt)", in1, in2, cr, rd2, approxT), nil } return tsNo, fmt.Sprintf("in=%d/%d cr1=%d cr2=%d rd=0 (~%dt)", in1, in2, cr, cr2, approxT), nil } return tsNo, "cr=0@all-levels(up to ~14400t, not supported or tier-limit)", nil } // probeCachingProvider 自适应探测 OpenRouter → Anthropic 路径的 prompt caching. // // 逻辑与 probeCachingAnthropic 相同,只是通过 flyto.ModelProvider 接口(EnableCaching=true) // 而非直连 api.Client.OpenRouter 将 cache_control 透传给 Anthropic. func probeCachingProvider(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { phrase := "You are a helpful AI assistant that answers concisely. " for _, reps := range []int{100, 300, 600, 1200} { system := strings.Repeat(phrase, reps) approxT := reps * 12 in1, cr, rd, err := cachingReqProvider(ctx, p, model, system) if err != nil { return tsError, fmt.Sprintf("err@~%dt(in=%d): %v", approxT, in1, err), err } // bug 修复(FIXED): 同上--第一次 cr=0 rd>0 = 缓存已存在,直接 ✓ if cr == 0 && rd > 0 { return tsYes, fmt.Sprintf("cache-hit-pre-existing in=%d cr=0 rd=%d (~%dt)", in1, rd, approxT), nil } if cr == 0 { continue } in2, cr2, rd2, err := cachingReqProvider(ctx, p, model, system) if err != nil { return tsError, fmt.Sprintf("cr=%d in1=%d err@2nd: %v", cr, in1, err), err } if rd2 > 0 { return tsYes, fmt.Sprintf("in=%d/%d cr=%d rd=%d (~%dt)", in1, in2, cr, rd2, approxT), nil } return tsNo, fmt.Sprintf("in=%d/%d cr1=%d cr2=%d rd=0 (~%dt)", in1, in2, cr, cr2, approxT), nil } return tsNo, "cr=0@all-levels(up to ~14400t, not supported or tier-limit)", nil } // probeCachingGeneric 通过 flyto.ModelProvider 发两次相同请求,检测 cache_read_tokens > 0. // 适用于自动 caching 的 provider(如 MiniMax 自有缓存机制). // // 升华改进(ELEVATED): 同 probeCachingAnthropic--新版从 UsageEvent.InputTokens 读 // 实际 input 数,打印到 diag,让 ✗ 判定可追溯. func probeCachingGeneric(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { doReq := func() (in, cr, rd int, err error) { reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() ch, e := p.Stream(reqCtx, &flyto.Request{ Model: model, MaxTokens: 5, System: "You are a helpful assistant. Always be concise.", Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Say hi.")}}, }, }) if e != nil { return 0, 0, 0, e } for evt := range ch { switch ev := evt.(type) { case *flyto.UsageEvent: in = ev.InputTokens cr = ev.CacheCreationTokens rd = ev.CacheReadTokens case *flyto.ErrorEvent: return 0, 0, 0, ev.Err } } return } in1, cr1, rd1, err := doReq() if err != nil { return tsError, fmt.Sprintf("err@1st: %v", err), err } // bug 修复(FIXED): 如果第一次就 rd>0,缓存已存在,直接返回 ✓ if rd1 > 0 { return tsYes, fmt.Sprintf("pre-existing in=%d cr=%d rd=%d", in1, cr1, rd1), nil } in2, _, rd2, err := doReq() if err != nil { return tsError, fmt.Sprintf("err@2nd: %v", err), err } if rd2 > 0 { return tsYes, fmt.Sprintf("auto-cache in=%d/%d cr=%d rd=%d", in1, in2, cr1, rd2), nil } return tsNo, fmt.Sprintf("no auto-cache in=%d/%d cr=%d rd=%d", in1, in2, cr1, rd2), nil } // probeSchemaRef 探测模型工具调用是否支持 JSON Schema $ref 引用. // // 精妙之处(CLEVER): $ref 是 JSON Schema 规范的合法特性,但部分模型(尤其是非 Anthropic 路由) // 在解析工具 InputSchema 时会直接报错或忽略 $ref,导致工具调用失败. // 探测方式:定义一个包含 $ref 的工具,检查模型能否正常返回 tool_use block. // // 反向思维:不能假设 $ref 展开由 provider 网关完成--OpenRouter 不展开 $ref, // 直接透传给底层模型,所以这个能力反映的是目标模型的真实支持情况. func probeSchemaRef(ctx context.Context, p flyto.ModelProvider, model string) (tristate, string, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() // 包含 $ref 的 schema:definitions 中定义 Location 类型,properties 通过 $ref 引用 // 替代方案:<内联展开 $ref,不测这个特性> - 否决: // 若工具 builder 生成的 schema 含 $ref,运行时会静默失败,提前探测更安全. schemaWithRef := json.RawMessage(`{ "type": "object", "definitions": { "Location": { "type": "object", "properties": { "city": {"type": "string"}, "country": {"type": "string"} }, "required": ["city"] } }, "properties": { "location": {"$ref": "#/definitions/Location"} }, "required": ["location"] }`) ch, err := p.Stream(ctx, &flyto.Request{ Model: model, MaxTokens: 256, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Get weather for Paris, France using get_weather.")}}, }, Tools: []flyto.Tool{ { Name: "get_weather", Description: "Get weather for a location", InputSchema: schemaWithRef, }, }, }) if err != nil { return tsError, fmt.Sprintf("connect err: %v", err), err } for evt := range ch { switch e := evt.(type) { case *flyto.ToolUseEvent: drain(ch) return tsYes, "", nil case *flyto.ErrorEvent: // 精妙之处(CLEVER): $ref 不支持时,Anthropic 等 provider 返回 400 invalid_request_error, // 错误信息通常含 "$ref" 或 "schema",可作为额外诊断依据. errMsg := fmt.Sprintf("%v", e.Err) return tsNo, fmt.Sprintf("tool_use failed (likely $ref unsupported): %s", errMsg), nil } } return tsNo, "no tool_use event (model may not parse $ref schema)", nil } // probeToolCount 通过二分法探测模型实际可处理的最大工具数量. // // 升华改进(ELEVATED): 早期方案不探测工具数量上限,遇到"too many tools"错误时无诊断信息-- // 不知道是 64 个还是 128 个,也不知道是 provider 限制还是模型限制. // 新版二分搜索:从 [1, maxProbeTools] 区间找到最大可用数量, // 输出精确数字(如 "max=64"),帮助调用方合理分批工具. // // 二分策略: // - 上界 maxProbeTools(默认 128, 调用方可经 ProbeOpts.MaxProbeTools 覆盖) // - 每次发包含 N 个工具的请求,N 个工具名为 tool_0..tool_{N-1} // - 模型正常返回(任何非错误响应)→ 此数量可用,尝试更多 // - 模型返回错误 → 此数量不可用,尝试更少 // - 返回最后一个成功的数量(0 表示连 1 个工具都失败) // // 反向思维:二分法假设"N 个可用则 =%d (upper bound not reached)", maxProbeTools), nil } // 二分搜索 [1, maxProbeTools-1] // 找到的 lo → exhaustive=true:lo 通过且 lo+1 拒绝,是确定上限. lo, hi := 1, maxProbeTools-1 for lo < hi { mid := (lo + hi + 1) / 2 if tryN(mid) { lo = mid } else { hi = mid - 1 } } return lo, true, fmt.Sprintf("max=%d", lo), nil } // probeMaxOutputTokens 探测模型单次响应输出 token 数实测上限. // // probeMaxOutputTokens probes the model's per-response output token cap. // // 用一个鼓励长输出的 prompt + 极大 max_tokens (128K), 让 server / model // 自行决定 cap. 收 UsageEvent 的 OutputTokens + StopReason: // - StopReason="max_tokens" 或 "length" → exhaustive=true, OutputTokens // 是 server 端确认上限 (再大 server 也不给). // - StopReason="end_turn" / "stop_sequence" → exhaustive=false, model // 自己停了, 实际上限可能 ≥ OutputTokens (没探到). // // Uses a long-output-encouraging prompt + very large max_tokens (128K), // letting server / model decide the cap. Collects UsageEvent's // OutputTokens + StopReason: // - StopReason="max_tokens" or "length" → exhaustive=true, OutputTokens // is the server-confirmed cap (server won't give more). // - StopReason="end_turn" / "stop_sequence" → exhaustive=false, model // stopped on its own; real cap may be ≥ OutputTokens (not probed). // // 设计依据见 ADR-0005 § Bug S follow-up. 字段 day-1 起在 flyto.ModelInfo // 但无 probe 工具, 各 provider 静态表填的多是文档值或推断值. r19/r20 实证 // MiniMax-M2.7-highspeed 引擎升级 max_tokens=64000 后 server 接受不拒绝, // 说明静态表 16K 可能低估 — 跑此 probe 验证. // // See ADR-0005 § Bug S follow-up. Field exists in flyto.ModelInfo since // day-1 but had no probe; provider static tables filled mostly from docs // or inference. r19/r20 confirmed MiniMax-M2.7-highspeed accepts // max_tokens=64000 from the engine upgrade path without rejection, so // the static-table 16K may underestimate — this probe verifies. func probeMaxOutputTokens(ctx context.Context, p flyto.ModelProvider, model string) (int, bool, string, error) { ctx, cancel := context.WithTimeout(ctx, 180*time.Second) defer cancel() // 鼓励 model 输出尽量长, 不允许早停. // Encourage the model to output as long as possible, no early stop. prompt := "Write the longest possible essay about the complete history " + "of human civilization, from prehistoric times through the modern " + "era. Cover every major civilization in detail (Mesopotamia, Egypt, " + "Indus Valley, China, Greece, Rome, medieval Europe, Islamic " + "caliphates, Mongol Empire, Ming/Qing China, European colonialism, " + "Industrial Revolution, World Wars, Cold War, post-Cold War). For " + "each, describe political structure, economy, technology, religion, " + "art, daily life, decline causes. Do NOT stop early. Do NOT " + "summarize. Continue writing until your output budget is exhausted. " + "Output AT LEAST 50000 tokens of detailed essay text." ch, err := p.Stream(ctx, &flyto.Request{ Model: model, MaxTokens: 128_000, // 极大值, 让 server / model 决定 cap Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock(prompt)}}, }, }) if err != nil { return 0, false, fmt.Sprintf("connect err: %v", err), err } var outputTokens int var stopReason string for evt := range ch { switch e := evt.(type) { case *flyto.UsageEvent: // OpenAI / Gemini wire 路径 emit, 含 StopReason. // OpenAI / Gemini wire path emits, includes StopReason. if e.OutputTokens > outputTokens { outputTokens = e.OutputTokens } if e.StopReason != "" { stopReason = e.StopReason } case *flyto.TurnEndEvent: // Anthropic / MiniMax (anthropic-compat) 路径 emit. 不含 // StopReason — 用 OutputTokens 推断 exhaustive. // Anthropic / MiniMax (anthropic-compat) path emits. No // StopReason — infer exhaustive from OutputTokens. if e.OutputTokens > outputTokens { outputTokens = e.OutputTokens } case *flyto.ErrorEvent: return outputTokens, false, fmt.Sprintf("err: %v out=%d stop=%s", e.Err, outputTokens, stopReason), nil } } if outputTokens == 0 { return 0, false, "no token usage event received (neither UsageEvent nor TurnEndEvent)", nil } // exhaustive 推断: // StopReason 显式表示触底 → exhaustive=true (优先用 StopReason) // OutputTokens 接近 MaxTokens (≥ 95%) → exhaustive=true (推断触底) // 否则 model 自停, 实际上限 ≥ OutputTokens (没探到) // // Exhaustive inference: // StopReason explicitly indicates truncation → exhaustive=true (prefer StopReason) // OutputTokens close to MaxTokens (≥ 95%) → exhaustive=true (inferred) // Otherwise model self-stopped; real cap ≥ OutputTokens (not probed) exhaustive := stopReason == "max_tokens" || stopReason == "length" if !exhaustive && outputTokens >= 128_000*95/100 { // MaxTokens=128_000 95% ≈ 121K, 超此值视作触底. exhaustive = true } note := fmt.Sprintf("output_tokens=%d stop_reason=%q", outputTokens, stopReason) return outputTokens, exhaustive, note, nil } // probeSchemaFeatures 探测模型对 JSON Schema 各特性的支持 (L1174). // // 升华改进(ELEVATED): 早期方案只测 $ref(SchemaRef),不知道 enum/nested/array/数值约束 // 这些 OpenAI/Anthropic 文档有提但 MiniMax 文档未覆盖的特性是否可用. // 新版逐项发请求,每种 schema 特性单独一个工具,API 接受则标 true. // // 精妙之处(CLEVER): 测的是"API 是否接受此 schema"而非"模型是否遵守约束". // 前者是确定性的(ErrorEvent → 拒绝),后者是统计性的(需多次采样), // 对 probe 工具来说确定性结果更有价值. // // 替代方案: <合并到 SchemaRef 一起测> - 否决: $ref 测试的是引用展开, // enum/数值约束测试的是 schema validation,机制不同,合并会丢失粒度. func probeSchemaFeatures(ctx context.Context, p flyto.ModelProvider, model string) map[string]bool { schemas := []struct { name string schema json.RawMessage }{ {"enum", json.RawMessage(`{"type":"object","properties":{"color":{"type":"string","enum":["red","green","blue"]}},"required":["color"]}`)}, {"nested_object", json.RawMessage(`{"type":"object","properties":{"addr":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}},"required":["addr"]}`)}, {"array", json.RawMessage(`{"type":"object","properties":{"items":{"type":"array","items":{"type":"string"}}},"required":["items"]}`)}, {"numeric_min_max", json.RawMessage(`{"type":"object","properties":{"n":{"type":"integer","minimum":1,"maximum":100}},"required":["n"]}`)}, } results := make(map[string]bool, len(schemas)) for _, s := range schemas { results[s.name] = trySchemaFeature(ctx, p, model, s.name, s.schema) } return results } // trySchemaFeature 发包含指定 schema 的单工具请求,检测 API 是否接受. func trySchemaFeature(ctx context.Context, p flyto.ModelProvider, model string, toolName string, schema json.RawMessage) bool { reqCtx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() ch, err := p.Stream(reqCtx, &flyto.Request{ Model: model, MaxTokens: 100, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock(fmt.Sprintf("Use the %s tool with any valid value.", toolName))}}, }, Tools: []flyto.Tool{{ Name: toolName, Description: fmt.Sprintf("Test tool for %s schema feature", toolName), InputSchema: schema, }}, }) if err != nil { return false } for evt := range ch { switch evt.(type) { case *flyto.ToolUseEvent: drain(ch) return true case *flyto.TextDeltaEvent, *flyto.TextEvent: // 模型用文本回复而非调用工具 -- schema 被 API 接受,只是模型选择不调用. drain(ch) return true case *flyto.ErrorEvent: drain(ch) return false } } return true // 流正常结束,无错误 } // drain 消费并丢弃 channel 剩余事件,防止 goroutine 泄漏. func drain(ch <-chan flyto.Event) { for range ch { } } // probeToolNameRegex 实测模型对工具名特殊字符 `.` 的接受度. // // 单点探测: 注册一个名为 "test.probe.tool" (含 `.`) 的工具, 触发模型 // 调用. 返回 "strict" 表示 4xx 拒 (regex 通常是 ^[a-zA-Z0-9_-]+$), // "permissive" 表示接受 (至少接受 `.`). // // 关键: 通过 req.Capabilities 显式 override ToolNameRegex 为 ".*" // 让 wire 层 ADR-0007 C4 的 ValidateToolNames pre-flight 不拦, 把 // 请求送到服务端, 测真协议而非测 wire 层. // // 完整 regex inference (`/`, `:`, 长度上限等) 留 follow-up — 单点 // 探测足以暴露 r22 类业务 bug (billcost.reflect 触发 OpenAI regex 拒). // // probeToolNameRegex empirically tests model acceptance of `.` in // tool names. Single-point probe: registers "test.probe.tool" and // triggers the model. Returns "strict" for 4xx rejection (regex // typically ^[a-zA-Z0-9_-]+$) or "permissive" for acceptance. // // CRITICAL: explicitly overrides req.Capabilities.ToolNameRegex to // ".*" so the wire-layer ADR-0007 C4 ValidateToolNames pre-flight // does NOT intercept — we want to test the real server protocol, not // our own wire layer's enforcement. func probeToolNameRegex(ctx context.Context, p flyto.ModelProvider, model string) (string, string, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() req := &flyto.Request{ Model: model, MaxTokens: 256, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock("Use the test.probe.tool with input {\"x\":1}.")}}, }, Tools: []flyto.Tool{ { Name: "test.probe.tool", Description: "ADR-0007 TD-20 capability prober — tool-name regex acceptance", InputSchema: json.RawMessage(`{"type":"object","properties":{"x":{"type":"integer"}}}`), }, }, // Override capabilities so the wire ValidateToolNames pre-flight // does not block this exact request. Mode/regex must be explicit // non-empty to bypass deepseek's resolveCapabilities fallback. // 显式 capability override, 防 wire 自身 pre-flight 拦截 + 防 // deepseek provider 的 capability fallback 回填. Capabilities: &flyto.ModelInfo{ ID: model, ToolNameRegex: `.*`, ReasoningPassbackMode: "none", ProviderKind: "direct", }, } ch, err := p.Stream(ctx, req) if err != nil { // Stream-level error before SSE — most likely server 4xx because // of the dotted tool name; treat as strict. // Stream 级错误 (SSE 之前) — 多半服务端 4xx 拒 dotted 工具名, 算 strict. return "strict", fmt.Sprintf("stream err: %v", err), nil } for evt := range ch { switch e := evt.(type) { case *flyto.ErrorEvent: drain(ch) return "strict", fmt.Sprintf("server rejected: %v", e.Err), nil case *flyto.ToolUseEvent: drain(ch) return "permissive", "tool_use returned with dotted name", nil } } // Stream 正常结束但模型没调工具: 不能断 strict (可能模型选择文本回复), // 也不能断 permissive (没收到 4xx 但也没 tool_use 证据). 返回 "" 让 // caller 看 note 判断. // Stream ended cleanly but no tool_use: cannot conclude strict // (model may have chosen text reply) or permissive (no 4xx but no // tool_use evidence). Return "" — caller reads the note. return "", "no tool_use, no error (model declined to call)", nil } // probeReasoningPassback 实测模型在多轮 tool calling 中是否要求 prior // assistant 的 reasoning_content 在下一轮 messages 中 passback. // // 2 round-trip 探测: // round 1: 发带 thinking + 1 tool 的 request, 让模型出 reasoning_content // + tool_use (前置条件: ToolUse=tsYes + Thinking=tsYes). // round 2: 发 tool_result, 但 assistant message **不**含 BlockThinking // (即不回传 reasoning_content). 看是否 4xx. // // 返回 "string" 表示 4xx 拒 (deepseek-v4-flash 文档明示 + r24 真因); // "none" 表示接受 (无需 passback). // // 关键: round 2 通过 req.Capabilities 显式 override // ReasoningPassbackMode="none" 防 wire 层 ADR-0007 C4 自动 inject // reasoning_content 掩盖真实协议. // // probeReasoningPassback empirically tests whether a model requires // the prior assistant turn's reasoning_content to be echoed in the // next request during multi-turn tool calling. // // Two round-trips: // round 1: send thinking + tool → model emits reasoning_content + // tool_use (precondition ToolUse=tsYes + Thinking=tsYes). // round 2: send tool_result with assistant message LACKING // BlockThinking (no reasoning passback) → check 4xx. // // Returns "string" for 4xx (deepseek-v4-flash docs + r24 root cause); // "none" for acceptance. // // CRITICAL: round 2 overrides req.Capabilities.ReasoningPassbackMode // to "none" so the wire layer's ADR-0007 C4 auto-inject does NOT mask // the real server protocol. func probeReasoningPassback(ctx context.Context, p flyto.ModelProvider, model string) (string, string, error) { ctx, cancel := context.WithTimeout(ctx, 90*time.Second) defer cancel() tools := []flyto.Tool{ { Name: "passback_probe", Description: "ADR-0007 TD-20 capability prober — reasoning_content passback", InputSchema: json.RawMessage(`{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}`), }, } userText := "Think step by step about what 17*23 is, then call the passback_probe tool with q=\"answer\"." round1Req := &flyto.Request{ Model: model, MaxTokens: 2048, NeedsThinking: true, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock(userText)}}, }, Tools: tools, Capabilities: &flyto.ModelInfo{ ID: model, ToolNameRegex: `.*`, ReasoningPassbackMode: "none", ProviderKind: "direct", }, } ch1, err := p.Stream(ctx, round1Req) if err != nil { return "", fmt.Sprintf("round1 stream err: %v", err), nil } var toolUseID, toolUseName string var toolUseInput map[string]any var sawReasoning bool for evt := range ch1 { switch e := evt.(type) { case *flyto.ThinkingEvent, *flyto.ThinkingDeltaEvent: sawReasoning = true case *flyto.ToolUseEvent: toolUseID = e.ID toolUseName = e.ToolName toolUseInput = e.Input if toolUseInput == nil { toolUseInput = map[string]any{} } case *flyto.ErrorEvent: drain(ch1) return "", fmt.Sprintf("round1 err: %v", e.Err), nil } } if !sawReasoning { return "", "round1 produced no reasoning (precondition failed)", nil } if toolUseID == "" { return "", "round1 produced no tool_use (precondition failed)", nil } // Round 2: assistant message contains tool_use but NO BlockThinking. // User message contains tool_result for the round-1 tool_use_id. // 关键: 不构造 BlockThinking, 让 reasoning_content 在 messages 里缺位, // 再用 capability override 防 wire 层自动 inject 兜底. round2Req := &flyto.Request{ Model: model, MaxTokens: 256, Messages: []flyto.Message{ {Role: flyto.RoleUser, Blocks: []flyto.Block{flyto.TextBlock(userText)}}, {Role: flyto.RoleAssistant, Blocks: []flyto.Block{ {Type: flyto.BlockToolUse, ToolUseID: toolUseID, ToolName: toolUseName, ToolInput: toolUseInput}, }}, {Role: flyto.RoleUser, Blocks: []flyto.Block{ {Type: flyto.BlockToolResult, ToolUseID: toolUseID, ResultText: "answer=391"}, }}, }, Tools: tools, Capabilities: &flyto.ModelInfo{ ID: model, ToolNameRegex: `.*`, ReasoningPassbackMode: "none", // explicit override prevents wire auto-inject ProviderKind: "direct", }, } ch2, err := p.Stream(ctx, round2Req) if err != nil { return "string", fmt.Sprintf("round2 stream err (likely 400 on missing reasoning): %v", err), nil } for evt := range ch2 { switch e := evt.(type) { case *flyto.ErrorEvent: drain(ch2) return "string", fmt.Sprintf("round2 4xx: %v", e.Err), nil } } return "none", "round2 accepted assistant message without reasoning_content", nil }