// transcription.go implements the shared flyto.TranscriptionProvider // capability for OpenAI-compatible endpoints via a single multipart POST to // /v1/audio/transcriptions. First implementer is a self-hosted oMLX ASR // server (model_type=audio_stt in its /v1/models catalog); the wire shape is // the OpenAI transcription API plus the server's extension form fields // (diarization, long-audio chunking, word timestamps), so it also works // against real OpenAI for the core fields. // // Why a direct POST and NOT Provider.Stream (mirrors vision.go): transcription // is a single-shot RPC, not the engine's multi-turn streaming loop. The audio // is STREAMED to the wire through an io.Pipe -- recordings run to hundreds of // MB and must not be buffered whole in memory. The pipe writer runs in its own // goroutine; a request-build error on that side is propagated through // CloseWithError so the HTTP client surfaces it instead of a silent short body. // // transcription.go 用 OpenAI 兼容端点实现共享 flyto.TranscriptionProvider: // 单次 multipart POST /v1/audio/transcriptions. 首个实现对象是自托管 oMLX ASR // 服务端 (其 /v1/models 目录 model_type=audio_stt); 线上形状 = OpenAI 转写 API // + 服务端扩展表单字段 (说话人分离 / 长音频切块 / 词级时间戳), 核心字段对真 // OpenAI 也成立. // // 为什么直连 POST 而非 Provider.Stream (对齐 vision.go): 转写是单次 RPC, 不是 // 引擎多轮流式 loop. 音频经 io.Pipe **流式**上线 -- 录音可达数百 MB, 不许整段 // 进内存. pipe writer 在独立 goroutine 跑; 其侧构造错误经 CloseWithError 传播, // HTTP client 会报错而非静默短 body. package openai import ( "context" "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/http" "strconv" "strings" "time" "git.flytoex.net/yuanwei/flyto-agent/core/pkg/flyto" ) // Provider satisfies the shared flyto.TranscriptionProvider capability. var _ flyto.TranscriptionProvider = (*Provider)(nil) // transcriptionPath is appended to cfg.BaseURL for the transcription RPC. // transcriptionPath 追加到 cfg.BaseURL 的转写路径. const transcriptionPath = "/v1/audio/transcriptions" // Transcribe implements flyto.TranscriptionProvider. Single multipart POST; // blocks until the full response body is read -- for long recordings the // server transcribes before answering, so callers must budget the context // deadline in minutes, not seconds. Model is REQUIRED (per-request selection, // mirrors vision). For response_format json / verbose_json (or empty) the // body is parsed into the structured response; for text / srt / vtt the raw // body lands in Text. // // Transcribe 实现 flyto.TranscriptionProvider. 单次 multipart POST, 阻塞至读完 // body -- 长录音服务端转完才应答, 调用方的 context deadline 要按分钟级预算. // Model **必填** (per-request 选, 对齐 vision). response_format 为 json / // verbose_json (或空) 时解析结构化响应; text / srt / vtt 时原始 body 放 Text. func (p *Provider) Transcribe(ctx context.Context, req *flyto.TranscriptionRequest) (*flyto.TranscriptionResponse, error) { if req == nil { return nil, errors.New("openai: nil TranscriptionRequest") } if req.Audio == nil { return nil, errors.New("openai: TranscriptionRequest.Audio is required") } if strings.TrimSpace(req.Model) == "" { return nil, errors.New("openai: transcription requires Model (no endpoint-locked default)") } if p.cfg.APIKey == "" { return nil, errors.New("openai: APIKey required for transcription") } // Pre-flight the backend when a health route is configured: self-hosted // ASR boxes are not always online, and a cheap GET failing fast beats // streaming hundreds of MB into a dead endpoint (PM: 使用前一定要探活). // // 配了健康路由就先探活: 自托管 ASR 机器不常在线, 便宜 GET 快速失败好过把 // 数百 MB 灌进死端点 (PM: 使用前一定要探活). if err := p.CheckHealth(ctx); err != nil { return nil, fmt.Errorf("openai: transcription pre-flight: %w", err) } filename := req.Filename if filename == "" { filename = "audio" } // Stream the multipart body through a pipe so the audio is never fully // buffered. Field order: scalar fields first, file part last -- servers // that stream-parse can act on parameters before draining the audio. // // multipart body 走 pipe 流式产出, 音频永不整段缓存. 字段顺序: 标量字段在 // 前, 文件 part 最后 -- 流式解析的服务端可在拉完音频前拿到参数. pr, pw := io.Pipe() mw := multipart.NewWriter(pw) go func() { err := func() error { fields := map[string]string{ "model": req.Model, "language": req.Language, "prompt": req.Prompt, "response_format": req.ResponseFormat, "long_audio": req.LongAudio, "diarize_backend": req.DiarizeBackend, "left_speaker": req.LeftSpeaker, "right_speaker": req.RightSpeaker, } if req.ChunkMinutes > 0 { fields["chunk_minutes"] = strconv.FormatFloat(req.ChunkMinutes, 'f', -1, 64) } if req.WordTimestamps != nil { // Tristate: explicit false MUST hit the wire (it disables the // server-default word aligner -- the long-audio escape hatch). // 三态: 显式 false 必须上线 (关掉服务端默认的 word 对齐器 -- // 长音频的出路). fields["word_timestamps"] = strconv.FormatBool(*req.WordTimestamps) } for k, v := range req.Extra { // Typed fields win only when actually SET; an empty typed // value must not shadow an Extra passthrough of the same name. // 类型化字段只在真赋值时优先; 空值不得遮蔽同名 Extra 透传. if fields[k] == "" && k != "file" { fields[k] = v } } for k, v := range fields { if v == "" { continue } if err := mw.WriteField(k, v); err != nil { return fmt.Errorf("write field %s: %w", k, err) } } fw, err := mw.CreateFormFile("file", filename) if err != nil { return fmt.Errorf("create file part: %w", err) } if _, err := io.Copy(fw, req.Audio); err != nil { return fmt.Errorf("copy audio: %w", err) } return mw.Close() }() // CloseWithError(nil) == Close: the reader sees EOF on success and // the build error otherwise. CloseWithError(nil) 等价 Close: 成功时 // reader 读到 EOF, 失败时读到构造错误. pw.CloseWithError(err) }() url := strings.TrimSuffix(p.cfg.BaseURL, "/") + transcriptionPath httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr) if err != nil { return nil, fmt.Errorf("openai: new transcription request: %w", err) } httpReq.Header.Set("Content-Type", mw.FormDataContentType()) httpReq.Header.Set("Authorization", "Bearer "+p.cfg.APIKey) // Transcription gets its OWN client, not the vision/chat one: the server // answers only after the FULL server-side job (which for long audio with // on_aligner_overflow=chunk means chunked re-ASR -- many minutes), so the // chat-scale ResponseHeaderTimeout default (seconds) would kill exactly // the long-audio requests this path exists for. The header timeout here // is a generous backstop; the real budget is the caller's ctx deadline. // // 转写用自己的 client, 不复用 vision/chat 的: 服务端做完**全部**活才应答 // (长音频带 on_aligner_overflow=chunk 是切块重转 -- 分钟级), chat 量级的 // ResponseHeaderTimeout 默认 (秒级) 会恰好杀掉本路径为之存在的长音频请求. // 这里的表头超时只是宽松兜底; 真预算是调用方 ctx deadline. resp, err := p.transcriptionHTTPClient().Do(httpReq) if err != nil { return nil, fmt.Errorf("openai: do transcription request: %w", err) } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("openai: read transcription response: %w", err) } if resp.StatusCode/100 != 2 { return nil, fmt.Errorf("openai: transcription status %d: %s", resp.StatusCode, visionSnippet(respBody, 400)) } // Non-JSON formats (text / srt / vtt) return the raw body as Text. // 非 JSON 格式 (text / srt / vtt) 原始 body 作 Text 返回. switch req.ResponseFormat { case "", "json", "verbose_json": default: return &flyto.TranscriptionResponse{Text: string(respBody)}, nil } var tr flyto.TranscriptionResponse if err := json.Unmarshal(respBody, &tr); err != nil { return nil, fmt.Errorf("openai: parse transcription response: %w (body=%s)", err, visionSnippet(respBody, 400)) } return &tr, nil } // transcriptionHTTPClient selects the client for the transcription RPC: an // injected cfg.HTTPClient wins (consumer owns timeouts); otherwise a client // whose ResponseHeaderTimeout covers minutes-scale server-side work, extended // further when the operator configured an even larger cfg.Timeout. // // transcriptionHTTPClient 选转写 RPC 的 client: 注入的 cfg.HTTPClient 优先 // (消费者自管超时); 否则表头超时按分钟级服务端工作量兜底, 操作员配了更大的 // cfg.Timeout 时随之放大. func (p *Provider) transcriptionHTTPClient() *http.Client { if p.cfg.HTTPClient != nil { return p.cfg.HTTPClient } timeout := 25 * time.Minute if p.cfg.Timeout > timeout { timeout = p.cfg.Timeout } return &http.Client{ Transport: &http.Transport{ ResponseHeaderTimeout: timeout, }, } }