// vision.go implements the shared flyto.VisionProvider capability for // OpenAI-compatible endpoints via a single non-streaming POST to // /v1/chat/completions carrying an image_url content block. This is the // SECOND vision implementer (after minimax) -- the moment the contract was // promoted from providers/minimax to flyto (see flyto/vision.go). // // Why a direct POST and NOT Provider.Stream: vision is a single-shot // extraction, not the engine's multi-turn streaming loop. The Stream path // still rejects BlockImage (its vision guard stays) -- that wiring is a // larger, future change. This keeps "extract one image" as its own seam, // symmetric with minimax's direct vlm RPC. OpenAI-compatible servers // (incl. a local oMLX gemma4 reached via OPENAI_BASE_URL) accept images as // {type:"image_url", image_url:{url:"data:;base64,"}} content. // // Reasoning models: oMLX gemma4 returns its chain-of-thought in // message.reasoning_content and the actual answer (the JSON) in // message.content. We read content; reasoning_content is surfaced only in // the error path (when content is empty) for diagnosis. defaultVisionMaxTokens // is generous because the reasoning alone can run thousands of tokens before // the answer (a low cap clips the JSON -- verified: image5 notice spent ~7700 // reasoning tokens before a 1200-char JSON answer). // // vision.go 用 OpenAI 兼容端点实现共享 flyto.VisionProvider: 单次非流式 POST // /v1/chat/completions 带 image_url content 块. 这是**第二个**视觉实现者 (在 // minimax 之后) -- 契约从 providers/minimax 提到 flyto 的时刻 (见 flyto/vision.go). // // 为什么直连 POST 而非 Provider.Stream: 视觉是单次抽取, 不是引擎的多轮流式 // loop. Stream 路径仍拒 BlockImage (其 vision guard 保留) -- 那条接线是更大的 // 将来改动. 这里把 "抽一张图" 留作独立接缝, 与 minimax 的直连 vlm RPC 对称. // OpenAI 兼容服务端 (含经 OPENAI_BASE_URL 接的本地 oMLX gemma4) 以 // {type:"image_url", image_url:{url:"data:;base64,"}} content 收图. // // 推理模型: oMLX gemma4 在 message.reasoning_content 返思维链, 真答案 (JSON) // 在 message.content. 我们读 content; reasoning_content 仅在错误路径 (content // 空时) 露出供诊断. defaultVisionMaxTokens 给得宽, 因为光推理就可能跑数千 // token 才出答案 (cap 太低切掉 JSON -- 实测: image5 通知推理 ~7700 token 才出 // 1200 字符 JSON). package openai import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "strings" "git.flytoex.net/yuanwei/flyto-agent/core/pkg/flyto" ) // Provider satisfies the shared flyto.VisionProvider capability. var _ flyto.VisionProvider = (*Provider)(nil) const ( // visionChatPath is appended to cfg.BaseURL for the chat/completions RPC. // visionChatPath 追加到 cfg.BaseURL 的 chat/completions 路径. visionChatPath = "/v1/chat/completions" // defaultVisionMediaType applies when VisionRequest.MediaType is blank. // defaultVisionMediaType 当 VisionRequest.MediaType 留空时生效. defaultVisionMediaType = "image/png" // defaultVisionMaxTokens caps generation when VisionRequest.MaxTokens is 0. // Sized for reasoning models that spend thousands of tokens reasoning before // the answer; a low cap clips the JSON. // // defaultVisionMaxTokens 当 VisionRequest.MaxTokens 为 0 时限生成. 为推理 // 模型留余量 (出答案前花数千 token 推理), cap 太低切掉 JSON. defaultVisionMaxTokens = 16384 ) // visionWireRequest mirrors the OpenAI chat/completions request with a // multimodal content array. stream is always false (single-shot extraction). // // visionWireRequest 镜像 OpenAI chat/completions 请求 (多模态 content 数组). // stream 恒 false (单次抽取). type visionWireRequest struct { Model string `json:"model"` Messages []visionWireMessage `json:"messages"` MaxTokens int `json:"max_tokens,omitempty"` Stream bool `json:"stream"` } type visionWireMessage struct { Role string `json:"role"` Content []visionWireContent `json:"content"` } type visionWireContent struct { Type string `json:"type"` Text string `json:"text,omitempty"` ImageURL *visionWireImageURL `json:"image_url,omitempty"` } type visionWireImageURL struct { URL string `json:"url"` // data:;base64, } // visionWireResponse mirrors the chat/completions response envelope. Only // choices[0].message.{content,reasoning_content} + the error envelope are // consumed; future fields are silently ignored. // // visionWireResponse 镜像 chat/completions 响应外壳. 只消费 choices[0]. // message.{content,reasoning_content} + error 外壳; 未来字段静默忽略. type visionWireResponse struct { Choices []struct { Message struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` } `json:"message"` FinishReason string `json:"finish_reason"` } `json:"choices"` Error *struct { Message string `json:"message"` Type string `json:"type"` } `json:"error"` } // ExtractVision implements flyto.VisionProvider. Single non-streaming HTTP // POST; blocks until the full response body is read. Model is REQUIRED (the // OpenAI wire has no endpoint-locked model like minimax). Errors propagate // from transport failures, non-2xx HTTP, an error envelope, or empty content // (caller cannot parse "" so we fail loudly, surfacing a reasoning snippet to // diagnose a clipped / reasoning-only response). // // ExtractVision 实现 flyto.VisionProvider. 单次非流式 POST, 阻塞至读完 body. // Model **必填** (OpenAI 线协议无 minimax 那种端点锁模型). 错误来源: 传输失败 // / 非 2xx / error 外壳 / 空 content (调用方解析 "" 没意义, fail-loud, 附 // reasoning 片段诊断被切断 / 只有推理的响应). func (p *Provider) ExtractVision(ctx context.Context, req *flyto.VisionRequest) (*flyto.VisionResponse, error) { if req == nil { return nil, errors.New("openai: nil VisionRequest") } if len(req.Image) == 0 { return nil, errors.New("openai: empty image bytes") } if p.cfg.APIKey == "" { return nil, errors.New("openai: APIKey required for vision") } if strings.TrimSpace(req.Model) == "" { return nil, errors.New("openai: vision requires Model (no endpoint-locked default)") } mediaType := req.MediaType if mediaType == "" { mediaType = defaultVisionMediaType } dataURI := "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(req.Image) maxTokens := req.MaxTokens if maxTokens == 0 { maxTokens = defaultVisionMaxTokens } body, err := json.Marshal(visionWireRequest{ Model: req.Model, MaxTokens: maxTokens, Stream: false, Messages: []visionWireMessage{{ Role: "user", Content: []visionWireContent{ {Type: "text", Text: req.Prompt}, {Type: "image_url", ImageURL: &visionWireImageURL{URL: dataURI}}, }, }}, }) if err != nil { return nil, fmt.Errorf("openai: marshal vision request: %w", err) } url := strings.TrimSuffix(p.cfg.BaseURL, "/") + visionChatPath httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("openai: new vision request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Authorization", "Bearer "+p.cfg.APIKey) resp, err := p.visionHTTPClient().Do(httpReq) if err != nil { return nil, fmt.Errorf("openai: do vision request: %w", err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("openai: read vision response: %w", err) } if resp.StatusCode/100 != 2 { return nil, fmt.Errorf("openai: vision status %d: %s", resp.StatusCode, visionSnippet(respBody, 400)) } var vr visionWireResponse if err := json.Unmarshal(respBody, &vr); err != nil { return nil, fmt.Errorf("openai: parse vision envelope: %w (body=%s)", err, visionSnippet(respBody, 400)) } if vr.Error != nil && vr.Error.Message != "" { return nil, fmt.Errorf("openai: vision error: %s", vr.Error.Message) } if len(vr.Choices) == 0 { return nil, fmt.Errorf("openai: vision no choices (body=%s)", visionSnippet(respBody, 400)) } content := vr.Choices[0].Message.Content if strings.TrimSpace(content) == "" { // Empty content with non-empty reasoning = the model reasoned but never // emitted an answer (max_tokens clip, or a reasoning-only response). // Surface a reasoning snippet so the caller can tell what happened. // 空 content + 非空 reasoning = 模型推理了但没出答案 (max_tokens 切断或 // 只返推理). 露 reasoning 片段让调用方判断. return nil, fmt.Errorf("openai: vision empty content (finish=%s, reasoning=%s)", vr.Choices[0].FinishReason, visionSnippet([]byte(vr.Choices[0].Message.ReasoningContent), 300)) } return &flyto.VisionResponse{Content: content}, nil } // visionHTTPClient picks the http.Client for vision calls. Reuses // cfg.HTTPClient if injected (test or custom transport); else builds one with // cfg.Timeout (or defaultTimeout) as ResponseHeaderTimeout. For a NON-stream // chat call the server sends the response only after full generation, so the // timeout must exceed total generation time -- callers wire a vision-friendly // timeout (the embedded-image extractor uses 180s). // // visionHTTPClient 选 vision 调用的 http.Client. 注入了 cfg.HTTPClient 则复用; // 否则按 cfg.Timeout (或 defaultTimeout) 当 ResponseHeaderTimeout 新建. 非流式 // chat 服务端**生成完才**发响应, 故 timeout 须大于总生成时间 -- 调用方接 vision // 友好超时 (内嵌图抽取器用 180s). func (p *Provider) visionHTTPClient() *http.Client { if p.cfg.HTTPClient != nil { return p.cfg.HTTPClient } timeout := p.cfg.Timeout if timeout == 0 { timeout = defaultTimeout } return &http.Client{ Transport: &http.Transport{ ResponseHeaderTimeout: timeout, }, } } // visionSnippet bounds error messages so an oversized upstream body does not // flood logs. 400 chars identifies the failure while keeping log lines bounded. // // visionSnippet 限 error 消息长度, 防超大上游 body 灌满日志. 400 字符足以识别 // 失败原因, log 行长可控. func visionSnippet(b []byte, n int) string { if len(b) <= n { return string(b) } return string(b[:n]) + "..." }