package validator import ( "context" "errors" "strings" "testing" "git.flytoex.net/yuanwei/flyto-agent/pkg/flyto" ) // -- parseLLMVerdict / extractJSONObject -- func TestParseLLMVerdict_PlainJSON(t *testing.T) { v, err := parseLLMVerdict(`{"approved": true, "score": 0.9, "severity": "warn", "reason": "ok"}`) if err != nil { t.Fatalf("parse: %v", err) } if !v.Approved { t.Errorf("should approve, got %+v", v) } if v.Score != 0.9 { t.Errorf("score mismatch: got %v", v.Score) } if v.Severity != SeverityWarn { t.Errorf("severity mismatch: got %v", v.Severity) } } func TestParseLLMVerdict_FencedJSON(t *testing.T) { resp := "```json\n" + `{"approved": true, "score": 1, "severity": "warn"}` + "\n```" v, err := parseLLMVerdict(resp) if err != nil { t.Fatalf("parse: %v", err) } if !v.Approved { t.Errorf("fenced JSON should parse, got %+v", v) } } func TestParseLLMVerdict_WithPreamble(t *testing.T) { resp := `After analysis, my verdict: {"approved": false, "score": 0.0, "severity": "block", "reason": "DROP detected"} end.` v, err := parseLLMVerdict(resp) if err != nil { t.Fatalf("parse: %v", err) } if v.Approved { t.Errorf("prose-wrapped JSON should parse and block, got %+v", v) } if v.Severity != SeverityBlock { t.Errorf("severity should be Block, got %v", v.Severity) } } func TestParseLLMVerdict_NestedObjects(t *testing.T) { resp := `{"approved": false, "score": 0.5, "severity": "block", "reason": "x", "details": {"rule1": {"ok": false}}}` v, err := parseLLMVerdict(resp) if err != nil { t.Fatalf("parse: %v", err) } if v.Details == nil { t.Fatalf("details should unmarshal, got nil") } inner, ok := v.Details["rule1"].(map[string]any) if !ok { t.Fatalf("nested rule1 should be map, got %T", v.Details["rule1"]) } if inner["ok"] != false { t.Errorf("nested ok should be false, got %v", inner["ok"]) } } func TestParseLLMVerdict_InvalidJSON(t *testing.T) { _, err := parseLLMVerdict(`not json at all`) if !errors.Is(err, ErrVerdictParse) { t.Errorf("expected ErrVerdictParse, got %v", err) } } func TestParseLLMVerdict_BracesInString(t *testing.T) { // Extractor must not confuse braces inside string literals. resp := `{"approved": true, "score": 0.5, "severity": "warn", "reason": "like {x} and }y{"}` v, err := parseLLMVerdict(resp) if err != nil { t.Fatalf("parse: %v", err) } if !strings.Contains(v.Reason, "{x}") { t.Errorf("braces in string should be preserved, got %q", v.Reason) } } func TestParseLLMVerdict_EscapedQuoteInString(t *testing.T) { // A backslash-escaped quote inside a string must not close the string. resp := `{"approved": true, "score": 0, "severity": "warn", "reason": "has \"quote\" inside"}` v, err := parseLLMVerdict(resp) if err != nil { t.Fatalf("parse: %v", err) } if !strings.Contains(v.Reason, `"quote"`) { t.Errorf("escaped quote should be preserved, got %q", v.Reason) } } func TestParseLLMVerdict_UnknownSeverity(t *testing.T) { // Unknown severity string is normalised to empty (Warn per convention). v, err := parseLLMVerdict(`{"approved": true, "score": 1, "severity": "critical"}`) if err != nil { t.Fatalf("parse: %v", err) } if v.Severity != "" { t.Errorf("unknown severity should reset to empty, got %v", v.Severity) } } // -- renderPrompt -- func TestLLMValidator_RenderPromptIncludesDiff(t *testing.T) { v := &LLMValidator{} prompt := v.renderPrompt(DiffInput{ SourceTool: "SQLCAS", Raw: []byte(`UPDATE orders SET status='x'`), Metadata: map[string]any{"table_name": "orders"}, }) if !strings.Contains(prompt, "SQLCAS") { t.Errorf("prompt should mention SourceTool, got:\n%s", prompt) } if !strings.Contains(prompt, "UPDATE orders") { t.Errorf("prompt should include Raw, got:\n%s", prompt) } if !strings.Contains(prompt, "table_name") { t.Errorf("prompt should include Metadata key, got:\n%s", prompt) } if !strings.Contains(prompt, "JSON") { t.Errorf("prompt should request JSON response, got:\n%s", prompt) } } func TestLLMValidator_RenderPromptEmptyMetadata(t *testing.T) { v := &LLMValidator{} prompt := v.renderPrompt(DiffInput{ SourceTool: "SQLDryRun", Raw: []byte(`{}`), }) if strings.Contains(prompt, "Metadata:") { t.Errorf("empty metadata should not emit Metadata line, got:\n%s", prompt) } } // -- Validate end-to-end (mock provider) -- func TestLLMValidator_ApproveVerdict(t *testing.T) { mp := &mockModelProvider{ events: []flyto.Event{ &flyto.TextEvent{Text: `{"approved": true, "score": 0.95, "severity": "warn", "reason": "looks safe"}`}, }, } client, _ := NewFlytoLLMClient(mp, "model-x", "be a reviewer") v := NewLLMValidator("llm-v1", "policy-alpha", client, "", 500) got, err := v.Validate(context.Background(), DiffInput{ SourceTool: "SQLCAS", Raw: []byte(`{"affected_rows": 1}`), }) if err != nil { t.Fatalf("unexpected error: %v", err) } if !got.Approved { t.Errorf("should approve, got %+v", got) } if got.Score != 0.95 { t.Errorf("score mismatch: got %v", got.Score) } if got.Reason != "looks safe" { t.Errorf("reason mismatch: got %q", got.Reason) } if got.ValidatorName != "llm-v1" { t.Errorf("ValidatorName should be stamped, got %q", got.ValidatorName) } if got.PolicyVersion != "policy-alpha" { t.Errorf("PolicyVersion should be stamped, got %q", got.PolicyVersion) } } func TestLLMValidator_BlockVerdict(t *testing.T) { mp := &mockModelProvider{ events: []flyto.Event{ &flyto.TextEvent{Text: `{"approved": false, "score": 0.1, "severity": "block", "reason": "drops table"}`}, }, } client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("llm-v1", "", client, "", 500) got, err := v.Validate(context.Background(), DiffInput{}) if err != nil { t.Fatalf("unexpected error: %v", err) } if got.Approved { t.Errorf("should block, got %+v", got) } if got.Severity != SeverityBlock { t.Errorf("severity should be Block, got %v", got.Severity) } } func TestLLMValidator_ValidatorIdentityOverwritesLLMSelfReport(t *testing.T) { // The LLM lies about ValidatorName / PolicyVersion; audit fields // must come from the Validator instance, not the response. mp := &mockModelProvider{ events: []flyto.Event{ &flyto.TextEvent{Text: `{"approved": true, "score": 1, "severity": "warn", "validator_name": "FAKE", "policy_version": "FAKE"}`}, }, } client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("real-validator", "real-v1", client, "", 500) got, _ := v.Validate(context.Background(), DiffInput{}) if got.ValidatorName != "real-validator" { t.Errorf("LLM self-report must be overridden, got %q", got.ValidatorName) } if got.PolicyVersion != "real-v1" { t.Errorf("LLM self-report must be overridden, got %q", got.PolicyVersion) } } func TestLLMValidator_ParseFailure(t *testing.T) { mp := &mockModelProvider{ events: []flyto.Event{ &flyto.TextEvent{Text: "I cannot answer that."}, }, } client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("llm-v1", "", client, "", 500) got, err := v.Validate(context.Background(), DiffInput{}) if !errors.Is(err, ErrVerdictParse) { t.Errorf("expected ErrVerdictParse, got %v", err) } if got.Approved { t.Errorf("parse failure should not approve, got %+v", got) } if got.Severity != SeverityBlock { t.Errorf("parse failure should be Block, got %v", got.Severity) } if _, ok := got.Details["raw_response"]; !ok { t.Errorf("parse failure Details should include raw_response for debugging") } } func TestLLMValidator_BackendFailure(t *testing.T) { mp := &mockModelProvider{err: errors.New("network down")} client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("llm-v1", "", client, "", 500) got, err := v.Validate(context.Background(), DiffInput{}) if !errors.Is(err, ErrValidatorBackend) { t.Errorf("expected ErrValidatorBackend, got %v", err) } if got.Approved { t.Errorf("backend failure should not approve, got %+v", got) } if got.Severity != SeverityBlock { t.Errorf("backend failure should be Block, got %v", got.Severity) } } func TestLLMValidator_CancelledContext(t *testing.T) { mp := &mockModelProvider{} client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("llm-v1", "", client, "", 500) ctx, cancel := context.WithCancel(context.Background()) cancel() got, err := v.Validate(ctx, DiffInput{}) if !errors.Is(err, context.Canceled) { t.Errorf("expected context.Canceled, got %v", err) } if got.Approved { t.Errorf("cancelled should not approve, got %+v", got) } if got.ValidatorName != "llm-v1" { t.Errorf("cancelled verdict should stamp identity, got %q", got.ValidatorName) } } func TestLLMValidator_EmptySeverityNormalisedToWarn(t *testing.T) { // LLM omits severity entirely; approved verdict must not land with // empty string (downstream circuit breaker parses Severity). mp := &mockModelProvider{ events: []flyto.Event{ &flyto.TextEvent{Text: `{"approved": true, "score": 1, "reason": "ok"}`}, }, } client, _ := NewFlytoLLMClient(mp, "model-x", "") v := NewLLMValidator("llm-v1", "", client, "", 500) got, _ := v.Validate(context.Background(), DiffInput{}) if got.Severity != SeverityWarn { t.Errorf("empty severity should normalise to Warn, got %v", got.Severity) } } func TestLLMValidator_Name(t *testing.T) { v := NewLLMValidator("my-llm", "", nil, "", 0) if v.Name() != "my-llm" { t.Errorf("Name mismatch: got %q", v.Name()) } }