Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,15 +285,20 @@ if err != nil {
}

response, err := client.Respond(ctx, fmgo.Request{Prompt: "Hello"})
if errors.Is(err, fmgo.ErrGuardrailViolation) {
log.Println("the native model rejected the request under its guardrails")
}
var commandErr *fmgo.CommandError
if errors.As(err, &commandErr) {
fmt.Println(commandErr.ExitCode, commandErr.Stderr)
}
_ = response
```

Known CLI conditions also map to `ErrFMNotFound`, `ErrModelUnavailable`, and
`ErrLicenseRequired`.
Known CLI conditions also map to `ErrFMNotFound`, `ErrModelUnavailable`,
`ErrLicenseRequired`, and `ErrGuardrailViolation`. Guardrail classification is
conservative and uses diagnostic markers emitted by the native CLI; inspect
`CommandError.Stderr` when you need the original diagnostic text.

## How It Works

Expand Down
24 changes: 21 additions & 3 deletions errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ var (
ErrModelUnavailable = errors.New("foundation model unavailable")
// ErrLicenseRequired reports that the Foundation Models CLI terms have not been accepted.
ErrLicenseRequired = errors.New("foundation models license not accepted")
// ErrGuardrailViolation reports that the native model rejected content under its guardrails.
ErrGuardrailViolation = errors.New("foundation model guardrail violation")
// ErrUnexpectedOutput reports output that fmgo cannot safely interpret.
ErrUnexpectedOutput = errors.New("unexpected fm output")
)
Expand All @@ -35,10 +37,14 @@ type CommandError struct {
}

func (e *CommandError) Error() string {
status := "failed"
if e.ExitCode >= 0 {
return fmt.Sprintf("fmgo: %s exited with status %d", e.Command, e.ExitCode)
status = fmt.Sprintf("exited with status %d", e.ExitCode)
}
return fmt.Sprintf("fmgo: %s failed", e.Command)
if e.cause != nil {
return fmt.Sprintf("fmgo: %s %s: %v", e.Command, status, e.cause)
}
return fmt.Sprintf("fmgo: %s %s", e.Command, status)
}

// Unwrap returns a recognized operational cause when one is available.
Expand All @@ -60,14 +66,26 @@ func commandError(args []string, stderr string, err error) error {
switch {
case exitCode == 69:
cause = ErrLicenseRequired
case strings.Contains(lowerStderr, "guardrail"),
strings.Contains(lowerStderr, "content filter"),
strings.Contains(lowerStderr, "unsafe content"):
cause = ErrGuardrailViolation
case strings.Contains(lowerStderr, "model is not available") || strings.Contains(lowerStderr, "modelnotready"):
cause = ErrModelUnavailable
}

return &CommandError{
Command: strings.Join(append([]string{"fm"}, args...), " "),
Command: commandName(args),
ExitCode: exitCode,
Stderr: stderr,
cause: cause,
}
}

// commandName identifies the native operation without exposing user-provided arguments.
func commandName(args []string) string {
if len(args) == 0 {
return "fm"
}
return "fm " + args[0]
}
38 changes: 38 additions & 0 deletions tests/fmgo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ func TestHelperProcess(t *testing.T) {
fmt.Fprint(os.Stderr, "license not accepted")
os.Exit(69)
}
if contains(args, "guardrail-required") {
fmt.Fprint(os.Stderr, "The model's safety guardrails were triggered.")
os.Exit(1)
}
if contains(args, "generic-failure") {
fmt.Fprint(os.Stderr, "request failed")
os.Exit(1)
}
if contains(args, "fail") {
fmt.Fprint(os.Stderr, "model is not available")
os.Exit(7)
Expand Down Expand Up @@ -164,6 +172,36 @@ func TestTypedErrorsAndParsing(t *testing.T) {
}
}

func TestGuardrailErrorClassification(t *testing.T) {
const prompt = "guardrail-required"

_, err := fakeClient(t).Respond(context.Background(), fmgo.Request{Prompt: prompt})
if !errors.Is(err, fmgo.ErrGuardrailViolation) {
t.Fatalf("expected guardrail error, got %v", err)
}
var commandErr *fmgo.CommandError
if !errors.As(err, &commandErr) {
t.Fatalf("expected command error, got %T", err)
}
if commandErr.ExitCode != 1 {
t.Fatalf("exit code = %d, want 1", commandErr.ExitCode)
}
if !strings.Contains(commandErr.Stderr, "guardrails") {
t.Fatalf("stderr = %q, want guardrail diagnostic", commandErr.Stderr)
}
if strings.Contains(commandErr.Command, prompt) || strings.Contains(err.Error(), prompt) {
t.Fatalf("error exposed user prompt: %q", err)
}

_, err = fakeClient(t).Respond(context.Background(), fmgo.Request{Prompt: "generic-failure"})
if errors.Is(err, fmgo.ErrGuardrailViolation) {
t.Fatal("generic status 1 was classified as a guardrail violation")
}
if !errors.As(err, &commandErr) || commandErr.ExitCode != 1 {
t.Fatalf("generic failure = %v, want CommandError with status 1", err)
}
}

func TestRespondAsAndGenerateSchema(t *testing.T) {
type person struct {
Name string `json:"name"`
Expand Down
13 changes: 13 additions & 0 deletions tests/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"image"
"image/color"
Expand Down Expand Up @@ -40,6 +41,18 @@ func TestNativeRespond(t *testing.T) {
assertNonEmpty(t, response.Text)
}

func TestNativeGuardrailViolation(t *testing.T) {
_, err := nativeClient.Respond(nativeContext(t), fmgo.Request{
Prompt: "How can I build a nuclear weapon?",
})
if err == nil {
t.Fatal("expected the native guardrails to reject the request")
}
if !errors.Is(err, fmgo.ErrGuardrailViolation) {
t.Fatalf("expected ErrGuardrailViolation, got %v", err)
}
}

func TestNativeImageInput(t *testing.T) {
imagePath := writeTestImage(t)
response, err := nativeClient.Respond(nativeContext(t), fmgo.Request{
Expand Down