diff --git a/.github/workflows/native-integration.yml b/.github/workflows/native-integration.yml new file mode 100644 index 0000000..1fefa87 --- /dev/null +++ b/.github/workflows/native-integration.yml @@ -0,0 +1,19 @@ +name: Native Integration + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + native-integration: + name: native-integration + runs-on: [self-hosted, macOS, ARM64, fmgo-native] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + - run: make integration diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba43b9a..7fdb102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,5 +19,4 @@ jobs: with: go-version-file: go.mod cache: false - - run: go test ./... - - run: go vet ./... + - run: make check diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d838da9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +examples/ diff --git a/AGENTS.md b/AGENTS.md index f2162e9..9e620e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,10 +43,10 @@ generic AI abstraction. ## Tests and Documentation - Use `WithExecutable` and the existing fake executable pattern for process - tests. Do not require a downloaded Foundation Model or run live prompts. + tests. Native tests run only with `FMGO_INTEGRATION=1` on a prepared machine. - Run `go fmt ./...`, `go vet ./...`, and `go test ./...` after Go changes. -- The full suite targets macOS 27 or later because the public constructor - validates platform support. +- Use `make integration` only on macOS 27+ Apple Silicon with the fm license + accepted and the system model available. - Update README, GoDoc, examples, and tests whenever a public contract changes. ## Workflow diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e84056..fb8202f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,14 +39,19 @@ refactor/process-cleanup Run these commands before opening a pull request: ```bash -go fmt ./... -go vet ./... -go test ./... +make check ``` The test suite uses a fake `fm` executable and does not require a Foundation -Model download. fmgo supports macOS 27 or later, so run the full suite on a -supported macOS version. +Model download. + +Native integration tests require macOS 27+ on Apple Silicon, the Foundation +Models CLI license accepted, and the system model available. Run them only on +a prepared machine: + +```bash +make integration +``` ## Commits diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..18f5738 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +GO ?= go + +.PHONY: help test integration vet examples check + +help: + @printf '%s\n' \ + 'make test Run the Go test suite.' \ + 'make integration Run native Foundation Models integration tests.' \ + 'make vet Run go vet.' \ + 'make examples Build integration examples into examples/.' \ + 'make check Run tests, vet, and build examples.' + +test: + $(GO) test ./... + +integration: + FMGO_INTEGRATION=1 $(GO) test -count=1 ./... + +vet: + $(GO) vet ./... + +examples: + mkdir -p examples + $(GO) build -o examples/respond ./cmd/respond + $(GO) build -o examples/server ./cmd/server + $(GO) build -o examples/stream ./cmd/stream + $(GO) build -o examples/structured ./cmd/structured + +check: test vet examples diff --git a/respond.go b/respond.go index a70b23a..626766d 100644 --- a/respond.go +++ b/respond.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "os" ) // Request configures a Foundation Models response. @@ -34,6 +35,12 @@ type Response struct { // Respond generates a complete non-streaming response. func (c *Client) Respond(ctx context.Context, request Request) (Response, error) { + request, cleanup, err := request.withSchemaFile() + if err != nil { + return Response{}, err + } + defer cleanup() + args, err := request.args(false) if err != nil { return Response{}, err @@ -45,6 +52,34 @@ func (c *Client) Respond(ctx context.Context, request Request) (Response, error) return Response{Text: string(stdout)}, nil } +func (r Request) withSchemaFile() (Request, func(), error) { + if len(r.Schema) == 0 { + return r, func() {}, nil + } + if r.SchemaFile != "" { + return Request{}, nil, fmt.Errorf("fmgo: schema and schema file cannot both be set") + } + + file, err := os.CreateTemp("", "fmgo-schema-*.json") + if err != nil { + return Request{}, nil, fmt.Errorf("fmgo: create schema file: %w", err) + } + path := file.Name() + if _, err := file.Write(r.Schema); err != nil { + _ = file.Close() + _ = os.Remove(path) + return Request{}, nil, fmt.Errorf("fmgo: write schema file: %w", err) + } + if err := file.Close(); err != nil { + _ = os.Remove(path) + return Request{}, nil, fmt.Errorf("fmgo: close schema file: %w", err) + } + + r.Schema = nil + r.SchemaFile = path + return r, func() { _ = os.Remove(path) }, nil +} + // RespondAs generates JSON for T and decodes it into T. func RespondAs[T any](ctx context.Context, client *Client, request Request) (T, error) { var result T diff --git a/schema.go b/schema.go index 3d25f22..2edda95 100644 --- a/schema.go +++ b/schema.go @@ -116,6 +116,9 @@ func SchemaFrom(value any) (Schema, error) { if err != nil { return nil, err } + if typeOf.Kind() == reflect.Struct && typeOf.Name() != "" { + schema["title"] = typeOf.Name() + } encoded, err := json.Marshal(schema) if err != nil { return nil, fmt.Errorf("fmgo: encode schema: %w", err) @@ -162,6 +165,7 @@ func schemaForStruct(typeOf reflect.Type, visiting map[reflect.Type]bool) (map[s properties := map[string]any{} required := []string{} + order := []string{} for index := range typeOf.NumField() { field := typeOf.Field(index) if !field.IsExported() || field.Anonymous { @@ -176,11 +180,17 @@ func schemaForStruct(typeOf reflect.Type, visiting map[reflect.Type]bool) (map[s return nil, err } properties[name] = property + order = append(order, name) if !optional { required = append(required, name) } } - schema := map[string]any{"type": "object", "properties": properties} + schema := map[string]any{ + "type": "object", + "x-order": order, + "properties": properties, + "additionalProperties": false, + } if len(required) != 0 { schema["required"] = required } diff --git a/stream.go b/stream.go index b53c03d..72dfb73 100644 --- a/stream.go +++ b/stream.go @@ -19,20 +19,29 @@ type Stream struct { text string err error once sync.Once + cleanup func() + clean sync.Once } // Stream starts a streaming response. func (c *Client) Stream(ctx context.Context, request Request) (*Stream, error) { + request, cleanup, err := request.withSchemaFile() + if err != nil { + return nil, err + } args, err := request.args(true) if err != nil { + cleanup() return nil, err } path, err := c.executablePath() if err != nil { + cleanup() return nil, err } command, err := internalprocess.StartPiped(ctx, path, args...) if err != nil { + cleanup() return nil, commandError(args, "", err) } stderr := make(chan []byte, 1) @@ -45,6 +54,7 @@ func (c *Client) Stream(ctx context.Context, request Request) (*Stream, error) { stdout: command.Stdout(), stderr: stderr, buffer: make([]byte, 32*1024), + cleanup: cleanup, }, nil } @@ -72,6 +82,7 @@ func (s *Stream) Next() bool { if waitErr := s.command.Wait(); waitErr != nil { s.err = commandError([]string{"respond", "--stream"}, string(stderr), waitErr) } + s.cleanupSchemaFile() return false } @@ -87,6 +98,11 @@ func (s *Stream) Close() error { s.once.Do(func() { _ = s.stdout.Close() closeErr = s.command.Close() + s.cleanupSchemaFile() }) return closeErr } + +func (s *Stream) cleanupSchemaFile() { + s.clean.Do(s.cleanup) +} diff --git a/tests/fmgo_test.go b/tests/fmgo_test.go index 3a6ed0d..3c52762 100644 --- a/tests/fmgo_test.go +++ b/tests/fmgo_test.go @@ -6,7 +6,9 @@ import ( "context" "errors" "fmt" + "io" "os" + "path/filepath" "strings" "testing" @@ -23,21 +25,48 @@ func TestHelperProcess(t *testing.T) { } switch args[0] { case "respond": + if contains(args, "block") { + select {} + } if contains(args, "--stream") { fmt.Fprint(os.Stdout, "hello") os.Exit(0) } + if contains(args, "license-required") { + fmt.Fprint(os.Stderr, "license not accepted") + os.Exit(69) + } if contains(args, "fail") { fmt.Fprint(os.Stderr, "model is not available") os.Exit(7) } + if os.Getenv("FMGO_HELPER_STRUCTURED_RESPONSE") == "1" { + fmt.Fprint(os.Stdout, `{"name":"Ada"}`) + os.Exit(0) + } fmt.Fprint(os.Stdout, strings.Join(args, "|")) case "count-tokens": + if contains(args, "invalid") { + fmt.Fprint(os.Stdout, "not-a-number\n") + os.Exit(0) + } fmt.Fprint(os.Stdout, "42\n") case "available": + if contains(args, "invalid") { + fmt.Fprint(os.Stdout, "unexpected") + os.Exit(0) + } fmt.Fprint(os.Stdout, "System model unavailable: modelNotReady\n") + case "schema": + fmt.Fprint(os.Stdout, `{"type":"object","properties":{"name":{"type":"string"}}}`) case "license": fmt.Fprint(os.Stdout, "Agreed to license FM1 version 1.0 on today.\n") + case "chat": + if os.Getenv("FMGO_HELPER_CHAT_ECHO") == "1" { + fmt.Fprint(os.Stdout, strings.Join(args, "|")) + os.Exit(0) + } + select {} default: os.Exit(2) } @@ -66,6 +95,35 @@ func TestRespondBuildsArgumentsWithoutShellInterpolation(t *testing.T) { } } +func TestRespondBuildsOptionalArguments(t *testing.T) { + response, err := fakeClient(t).Respond(context.Background(), fmgo.Request{ + Prompt: "hello", + SchemaFile: "schema.json", + Resume: "resume.json", + SaveTranscript: "transcript.json", + Verbose: true, + UseCase: fmgo.UseCaseContentTagging, + Guardrails: fmgo.GuardrailPermissiveContentTransformations, + Tools: []fmgo.Tool{fmgo.ToolBarcode}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "--schema|schema.json", + "--resume|resume.json", + "--save-transcript|transcript.json", + "--verbose", + "--use-case|content-tagging", + "--guardrails|permissive-content-transformations", + "--tool|barcode", + } { + if !strings.Contains(response.Text, want) { + t.Fatalf("response %q does not contain %q", response.Text, want) + } + } +} + func TestStream(t *testing.T) { stream, err := fakeClient(t).Stream(context.Background(), fmgo.Request{Prompt: "hello"}) if err != nil { @@ -98,6 +156,193 @@ func TestTypedErrorsAndParsing(t *testing.T) { if err != nil || !license.Accepted { t.Fatalf("license %#v, err %v", license, err) } + if _, err := client.ShowLicense(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := client.CountTokens(context.Background(), fmgo.TokenRequest{Prompt: "invalid"}); !errors.Is(err, fmgo.ErrUnexpectedOutput) { + t.Fatalf("expected unexpected output error, got %v", err) + } +} + +func TestRespondAsAndGenerateSchema(t *testing.T) { + type person struct { + Name string `json:"name"` + } + + t.Setenv("FMGO_HELPER_STRUCTURED_RESPONSE", "1") + client := fakeClient(t) + schema, err := client.GenerateSchema(context.Background(), fmgo.ObjectSchema{ + Name: "Person", + Properties: []fmgo.SchemaProperty{ + {Name: "name", Type: fmgo.SchemaString}, + {Name: "age", Type: fmgo.SchemaInteger, Optional: true}, + {Name: "score", Type: fmgo.SchemaDouble}, + {Name: "active", Type: fmgo.SchemaBoolean}, + {Name: "address", Type: fmgo.SchemaObject, Schema: fmgo.Schema(`{}`)}, + {Name: "result", Type: fmgo.SchemaAnyOf, Choices: []fmgo.Schema{fmgo.Schema(`{}`)}}, + }, + }) + if err != nil || len(schema) == 0 { + t.Fatalf("schema %s, err %v", schema, err) + } + + value, err := fmgo.RespondAs[person](context.Background(), client, fmgo.Request{Prompt: "person"}) + if err != nil { + t.Fatal(err) + } + if value.Name != "Ada" { + t.Fatalf("name = %q, want Ada", value.Name) + } +} + +func TestTokenRequestAndSchemaHelpers(t *testing.T) { + count, err := fakeClient(t).CountTokens(context.Background(), fmgo.TokenRequest{ + Prompt: "hello", + Instructions: "be concise", + Text: []string{"additional text"}, + Images: []string{"image.png"}, + Transcript: "transcript.json", + }) + if err != nil || count != 42 { + t.Fatalf("count %d, err %v", count, err) + } + + if _, err := fmgo.ParseSchema([]byte(`{"type":"object"}`)); err != nil { + t.Fatal(err) + } + if _, err := fmgo.ParseSchema([]byte(`not-json`)); err == nil { + t.Fatal("expected invalid schema error") + } + type record struct { + Name string `json:"name"` + } + if _, err := fmgo.SchemaFrom(record{}); err != nil { + t.Fatal(err) + } +} + +func TestRequestValidationAndTypedErrors(t *testing.T) { + client := fakeClient(t) + _, err := client.Respond(context.Background(), fmgo.Request{ + Images: []string{"photo.png"}, + ImageLabels: []string{"first", "second"}, + }) + if err == nil { + t.Fatal("expected image label validation error") + } + _, err = client.Respond(context.Background(), fmgo.Request{ + Schema: fmgo.Schema(`{}`), + SchemaFile: "schema.json", + }) + if err == nil { + t.Fatal("expected schema conflict error") + } + _, err = client.Respond(context.Background(), fmgo.Request{Prompt: "license-required"}) + if !errors.Is(err, fmgo.ErrLicenseRequired) { + t.Fatalf("expected license error, got %v", err) + } + var commandError *fmgo.CommandError + if !errors.As(err, &commandError) || commandError.ExitCode != 69 { + t.Fatalf("expected command error with exit code 69, got %v", err) + } + missingClient, err := fmgo.New(fmgo.WithExecutable(filepath.Join(t.TempDir(), "missing-fm"))) + if err != nil { + t.Fatal(err) + } + _, err = missingClient.Respond(context.Background(), fmgo.Request{Prompt: "hello"}) + if !errors.Is(err, fmgo.ErrFMNotFound) { + t.Fatalf("expected missing executable error, got %v", err) + } +} + +func TestStreamCancellationAndChatValidation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + stream, err := fakeClient(t).Stream(ctx, fmgo.Request{Prompt: "block"}) + if err != nil { + t.Fatal(err) + } + cancel() + defer stream.Close() + if stream.Next() { + t.Fatal("stream produced output after cancellation") + } + if stream.Err() == nil { + t.Fatal("expected stream cancellation error") + } + + _, err = fakeClient(t).StartChat(context.Background(), fmgo.ChatOptions{ + Continue: true, + Resume: "session", + }) + if err == nil { + t.Fatal("expected conflicting chat option error") + } +} + +func TestChatArguments(t *testing.T) { + t.Setenv("FMGO_HELPER_CHAT_ECHO", "1") + session, err := fakeClient(t).StartChat(context.Background(), fmgo.ChatOptions{ + Model: fmgo.ModelSystem, + Instructions: "be concise", + Resume: "session", + SetDefaultModel: fmgo.ModelSystem, + Tools: []fmgo.Tool{fmgo.ToolBarcode, fmgo.ToolOCR}, + }) + if err != nil { + t.Fatal(err) + } + defer session.Close() + output, err := io.ReadAll(session.Stdout()) + if err != nil { + t.Fatal(err) + } + if err := session.Wait(); err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "--model|system", + "--instructions|be concise", + "--resume|session", + "--set-default-model|system", + "--tool|barcode", + "--tool|ocr", + } { + if !strings.Contains(string(output), want) { + t.Fatalf("chat output %q does not contain %q", output, want) + } + } + + continued, err := fakeClient(t).StartChat(context.Background(), fmgo.ChatOptions{Continue: true}) + if err != nil { + t.Fatal(err) + } + defer continued.Close() + continuedOutput, err := io.ReadAll(continued.Stdout()) + if err != nil { + t.Fatal(err) + } + if err := continued.Wait(); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(continuedOutput), "--continue") { + t.Fatalf("chat output %q does not contain --continue", continuedOutput) + } +} + +func TestServerValidation(t *testing.T) { + client := fakeClient(t) + _, err := client.Serve(context.Background(), fmgo.ServerOptions{Port: 0}) + if err == nil { + t.Fatal("expected invalid port error") + } + _, err = client.Serve(context.Background(), fmgo.ServerOptions{ + Host: "127.0.0.1", + Port: 8080, + Socket: "fm.sock", + }) + if err == nil { + t.Fatal("expected conflicting socket error") + } } func TestSchemaFor(t *testing.T) { @@ -115,7 +360,15 @@ func TestSchemaFor(t *testing.T) { t.Fatal(err) } text := string(schema) - for _, want := range []string{`"name"`, `"address"`, `"city"`, `"tags"`, `"required":["name","address"]`} { + for _, want := range []string{ + `"title":"Person"`, + `"additionalProperties":false`, + `"name"`, + `"address"`, + `"city"`, + `"tags"`, + `"required":["name","address"]`, + } { if !strings.Contains(text, want) { t.Fatalf("schema %s does not contain %s", text, want) } diff --git a/tests/integration_test.go b/tests/integration_test.go new file mode 100644 index 0000000..2193e7c --- /dev/null +++ b/tests/integration_test.go @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "image" + "image/color" + "image/png" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ruanklein/fmgo/v1" +) + +const nativeOperationTimeout = 90 * time.Second + +func TestNativeRespond(t *testing.T) { + response, err := nativeClient.Respond(nativeContext(t), fmgo.Request{ + Prompt: "Reply with a concise greeting.", + Instructions: "Use one sentence.", + Model: fmgo.ModelSystem, + Text: []string{"This is an integration test."}, + Greedy: true, + UseCase: fmgo.UseCaseGeneral, + Guardrails: fmgo.GuardrailDefault, + }) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, response.Text) +} + +func TestNativeImageInput(t *testing.T) { + imagePath := writeTestImage(t) + response, err := nativeClient.Respond(nativeContext(t), fmgo.Request{ + Prompt: "Describe the image in one sentence.", + Images: []string{imagePath}, + ImageLabels: []string{"test-image"}, + Tools: []fmgo.Tool{fmgo.ToolOCR}, + }) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, response.Text) +} + +func TestNativeStream(t *testing.T) { + stream, err := nativeClient.Stream(nativeContext(t), fmgo.Request{ + Prompt: "Reply with a short greeting.", + }) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + + var output strings.Builder + for stream.Next() { + output.WriteString(stream.Text()) + } + if err := stream.Err(); err != nil { + t.Fatal(err) + } + assertNonEmpty(t, output.String()) +} + +func TestNativeStructuredOutput(t *testing.T) { + type person struct { + Name string `json:"name"` + Age int `json:"age"` + } + + value, err := fmgo.RespondAs[person](nativeContext(t), nativeClient, fmgo.Request{ + Prompt: "Generate a fictional person with a name and a positive integer age.", + }) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, value.Name) + if value.Age <= 0 { + t.Fatalf("age = %d, want a positive value", value.Age) + } +} + +func TestNativeSchemaFile(t *testing.T) { + type person struct { + Name string `json:"name"` + } + + schema, err := fmgo.SchemaFor[person]() + if err != nil { + t.Fatal(err) + } + schemaPath := filepath.Join(t.TempDir(), "person.schema.json") + if err := os.WriteFile(schemaPath, schema, 0o600); err != nil { + t.Fatal(err) + } + + response, err := nativeClient.Respond(nativeContext(t), fmgo.Request{ + Prompt: "Generate a fictional person with a name.", + SchemaFile: schemaPath, + }) + if err != nil { + t.Fatal(err) + } + var value person + if err := json.Unmarshal([]byte(response.Text), &value); err != nil { + t.Fatalf("decode response: %v", err) + } + assertNonEmpty(t, value.Name) +} + +func TestNativeGenerateSchema(t *testing.T) { + schema, err := nativeClient.GenerateSchema(nativeContext(t), fmgo.ObjectSchema{ + Name: "Person", + Properties: []fmgo.SchemaProperty{ + {Name: "name", Type: fmgo.SchemaString}, + {Name: "age", Type: fmgo.SchemaInteger, Optional: true}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !json.Valid(schema) { + t.Fatalf("schema is not valid JSON: %s", schema) + } +} + +func TestNativeTranscriptAndTokenCount(t *testing.T) { + transcriptPath := filepath.Join(t.TempDir(), "conversation.json") + response, err := nativeClient.Respond(nativeContext(t), fmgo.Request{ + Prompt: "Reply with a concise greeting.", + SaveTranscript: transcriptPath, + }) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, response.Text) + + info, err := os.Stat(transcriptPath) + if err != nil { + t.Fatal(err) + } + if info.Size() == 0 { + t.Fatal("transcript is empty") + } + + count, err := nativeClient.CountTokens(nativeContext(t), fmgo.TokenRequest{ + Transcript: transcriptPath, + }) + if err != nil { + t.Fatal(err) + } + if count <= 0 { + t.Fatalf("token count = %d, want a positive value", count) + } + + resumed, err := nativeClient.Respond(nativeContext(t), fmgo.Request{ + Prompt: "Reply with a different concise greeting.", + Resume: transcriptPath, + }) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, resumed.Text) +} + +func TestNativeCountTokens(t *testing.T) { + count, err := nativeClient.CountTokens(nativeContext(t), fmgo.TokenRequest{ + Prompt: "Count these tokens.", + Instructions: "Be concise.", + Text: []string{"Additional text."}, + }) + if err != nil { + t.Fatal(err) + } + if count <= 0 { + t.Fatalf("token count = %d, want a positive value", count) + } +} + +func TestNativeAvailabilityAndLicense(t *testing.T) { + availability, err := nativeClient.Available(nativeContext(t)) + if err != nil { + t.Fatal(err) + } + if len(availability.Models) == 0 { + t.Fatal("availability did not report any models") + } + + license, err := nativeClient.License(nativeContext(t)) + if err != nil { + t.Fatal(err) + } + if !license.Accepted { + t.Fatal("license is not accepted") + } + + terms, err := nativeClient.ShowLicense(nativeContext(t)) + if err != nil { + t.Fatal(err) + } + assertNonEmpty(t, terms) +} + +func TestNativeChatLifecycle(t *testing.T) { + session, err := nativeClient.StartChat(nativeContext(t), fmgo.ChatOptions{ + Model: fmgo.ModelSystem, + Instructions: "Be concise.", + Tools: []fmgo.Tool{fmgo.ToolOCR}, + }) + if err != nil { + t.Fatal(err) + } + if session.Stdin() == nil || session.Stdout() == nil || session.Stderr() == nil { + t.Fatal("chat session did not expose all standard streams") + } + if err := session.Close(); err != nil { + t.Fatal(err) + } + if err := session.Close(); err != nil { + t.Fatal(err) + } +} + +func TestNativeServeTCP(t *testing.T) { + port := availableTCPPort(t) + server, err := nativeClient.Serve(nativeContext(t), fmgo.ServerOptions{ + Host: "127.0.0.1", + Port: port, + StartupTimeout: 30 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer server.Close() + + client := &http.Client{Timeout: nativeOperationTimeout} + assertHealth(t, client, "http://"+server.Addr()) + assertCompletion(t, client, "http://"+server.Addr(), false) + assertCompletion(t, client, "http://"+server.Addr(), true) +} + +func TestNativeServeUnixSocket(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "fm.sock") + server, err := nativeClient.Serve(nativeContext(t), fmgo.ServerOptions{ + Socket: socketPath, + StartupTimeout: 30 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer server.Close() + + dialer := net.Dialer{} + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, "unix", socketPath) + }, + } + client := &http.Client{Transport: transport, Timeout: nativeOperationTimeout} + defer transport.CloseIdleConnections() + assertHealth(t, client, "http://fm") +} + +func TestNativeServerValidation(t *testing.T) { + _, err := nativeClient.Serve(nativeContext(t), fmgo.ServerOptions{Port: 0}) + if err == nil { + t.Fatal("Serve accepted port zero") + } + _, err = nativeClient.Serve(nativeContext(t), fmgo.ServerOptions{ + Host: "127.0.0.1", + Port: 8080, + Socket: filepath.Join(t.TempDir(), "fm.sock"), + }) + if err == nil { + t.Fatal("Serve accepted TCP and Unix socket options together") + } +} + +func nativeContext(t *testing.T) context.Context { + t.Helper() + if nativeClient == nil { + t.Skip("native integration requires FMGO_INTEGRATION=1") + } + ctx, cancel := context.WithTimeout(context.Background(), nativeOperationTimeout) + t.Cleanup(cancel) + return ctx +} + +func writeTestImage(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "image.png") + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = file.Close() }) + + picture := image.NewRGBA(image.Rect(0, 0, 32, 32)) + for y := range 32 { + for x := range 32 { + picture.Set(x, y, color.RGBA{R: 0, G: 128, B: 255, A: 255}) + } + } + if err := png.Encode(file, picture); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func availableTCPPort(t *testing.T) int { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + return listener.Addr().(*net.TCPAddr).Port +} + +func assertHealth(t *testing.T, client *http.Client, address string) { + t.Helper() + response, err := client.Get(address + "/health") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("health status = %s", response.Status) + } +} + +func assertCompletion(t *testing.T, client *http.Client, address string, stream bool) { + t.Helper() + payload := fmt.Sprintf(`{"model":"system","messages":[{"role":"user","content":"Reply with a concise greeting."}],"stream":%t}`, stream) + request, err := http.NewRequest(http.MethodPost, address+"/v1/chat/completions", strings.NewReader(payload)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + body, _ := io.ReadAll(response.Body) + t.Fatalf("completion status = %s: %s", response.Status, body) + } + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if len(bytes.TrimSpace(body)) == 0 { + t.Fatal("completion body is empty") + } + if stream { + return + } + var completion struct { + Choices []json.RawMessage `json:"choices"` + } + if err := json.Unmarshal(body, &completion); err != nil { + t.Fatalf("decode completion: %v", err) + } + if len(completion.Choices) == 0 { + t.Fatal("completion did not include choices") + } +} + +func assertNonEmpty(t *testing.T, value string) { + t.Helper() + if strings.TrimSpace(value) == "" { + t.Fatal("received an empty response") + } +} diff --git a/tests/main_test.go b/tests/main_test.go new file mode 100644 index 0000000..80651d6 --- /dev/null +++ b/tests/main_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/ruanklein/fmgo/v1" +) + +const nativeSetupTimeout = 30 * time.Second + +var nativeClient *fmgo.Client + +func TestMain(m *testing.M) { + if os.Getenv("FMGO_HELPER") == "1" { + os.Exit(m.Run()) + } + if os.Getenv("FMGO_INTEGRATION") != "1" { + os.Exit(m.Run()) + } + + client, err := fmgo.New() + if err != nil { + fmt.Fprintf(os.Stderr, "fmgo integration setup: %v\n", err) + os.Exit(1) + } + + ctx, cancel := context.WithTimeout(context.Background(), nativeSetupTimeout) + defer cancel() + if err := validateNativeEnvironment(ctx, client); err != nil { + fmt.Fprintf(os.Stderr, "fmgo integration setup: %v\n", err) + os.Exit(1) + } + + nativeClient = client + os.Exit(m.Run()) +} + +func validateNativeEnvironment(ctx context.Context, client *fmgo.Client) error { + license, err := client.License(ctx) + if err != nil { + return fmt.Errorf("check license: %w", err) + } + if !license.Accepted { + return fmt.Errorf("Foundation Models license is not accepted") + } + + availability, err := client.Available(ctx) + if err != nil { + return fmt.Errorf("check model availability: %w", err) + } + for _, model := range availability.Models { + if model.Model == fmgo.ModelSystem && model.Available { + return nil + } + } + return fmt.Errorf("system Foundation Model is unavailable") +}