diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2339523..00ebef4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,14 +196,19 @@ jobs: else BUILT_LIB="sproink-build/target/release/libsproink.a" fi - # Build only if cached artifact is absent + # Build only if cached artifact is absent. + # + # sproink's Cargo.toml narrows [lib] crate-type to ["lib"], so plain + # `cargo build` produces only an rlib. Use `cargo rustc` to explicitly + # request a staticlib, and enable the `ffi` feature to gate-in the + # `pub mod ffi` module that exposes the extern "C" symbols. if [ ! -f "$BUILT_LIB" ]; then cd sproink-build if [ "$RUNNER_OS" = "Windows" ]; then rustup target add x86_64-pc-windows-gnu - cargo build --release --target x86_64-pc-windows-gnu + cargo rustc --release --lib --crate-type staticlib --features ffi --target x86_64-pc-windows-gnu else - cargo build --release + cargo rustc --release --lib --crate-type staticlib --features ffi fi cd .. fi diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index 756f1242..c3eaf302 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -98,12 +98,49 @@ jobs: rm -f lancedb-go-native-binaries.tar.gz fi + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache sproink build + uses: actions/cache@v5 + with: + path: sproink-build/target + key: sproink-test-release-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('third_party/sproink/include/sproink.h') }} + + - name: Build sproink native library + shell: bash + run: | + # Clone source if missing (cache only restores target/, not source). + if [ ! -f "sproink-build/Cargo.toml" ]; then + git clone --depth 1 https://github.com/nvandessel/sproink.git sproink-src-tmp + if [ -d "sproink-build/target" ]; then + mv sproink-build/target sproink-src-tmp/target + fi + rm -rf sproink-build + mv sproink-src-tmp sproink-build + fi + BUILT_LIB="sproink-build/target/release/libsproink.a" + # sproink's Cargo.toml narrows [lib] crate-type to ["lib"]; use cargo + # rustc to explicitly request a staticlib and enable the `ffi` feature + # that gates `pub mod ffi`. + if [ ! -f "$BUILT_LIB" ]; then + cd sproink-build + cargo rustc --release --lib --crate-type staticlib --features ffi + cd .. + fi + mkdir -p third_party/sproink/lib/linux_amd64 + cp "$BUILT_LIB" third_party/sproink/lib/linux_amd64/libsproink.a + - name: Build with CGO + LanceDB run: | PLATFORM="$(go env GOOS)_$(go env GOARCH)" + # liblancedb_go.a and libsproink.a are both Rust staticlibs and each + # bundle their own copy of libstd. --allow-multiple-definition lets + # the linker keep the first definition for the duplicate libstd + # symbols (rust_eh_personality, etc.). CGO_ENABLED=1 \ CGO_CFLAGS="-I$(pwd)/include" \ - CGO_LDFLAGS="-L$(pwd)/lib/${PLATFORM} -Wl,-Bstatic -llancedb_go -Wl,-Bdynamic -lm -ldl -lstdc++ -lpthread" \ + CGO_LDFLAGS="-L$(pwd)/lib/${PLATFORM} -Wl,-Bstatic -llancedb_go -Wl,-Bdynamic -lm -ldl -lstdc++ -lpthread -Wl,--allow-multiple-definition" \ go build -ldflags "-s -w -X main.version=test-cgo" \ -o floop-cgo ./cmd/floop diff --git a/.golangci.yml b/.golangci.yml index bd26d860..98ad46ec 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,6 @@ linters: - unused - gosec - misspell - - goconst disable: - lll - gocyclo @@ -36,15 +35,12 @@ linters: - G302 # File permissions — tracked in w03.6 - G304 # File path from variable — we have pathutil validation - G306 # WriteFile permissions — tracked in w03.6 - goconst: - min-occurrences: 10 exclusions: rules: - path: _test\.go linters: - errcheck - gosec - - goconst # Ignore errcheck on defer Close() — standard Go pattern - text: "Error return value of .*.Close.*is not checked" linters: diff --git a/cmd/floop/cmd_vault.go b/cmd/floop/cmd_vault.go new file mode 100644 index 00000000..c66d14da --- /dev/null +++ b/cmd/floop/cmd_vault.go @@ -0,0 +1,496 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/nvandessel/floop/internal/config" + "github.com/nvandessel/floop/internal/store" + "github.com/nvandessel/floop/internal/vault" + "github.com/spf13/cobra" +) + +// defaultVaultDims is the default vector dimensions for vault sync. +// This should match the embedding model dimensions used by the system. +const defaultVaultDims = 768 + +func newVaultCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "vault", + Short: "Lance-native S3 backup and sync", + Long: `Manage vault sync for floop's behavioral memory store. + +Vault sync provides bidirectional synchronization between local floop stores +and a remote S3-compatible backend (MinIO, AWS S3, R2). + +Commands: + init Configure vault remote and test connectivity + push Push local state to remote + pull Pull remote state to local + sync Bidirectional sync (pull then push) + status Show sync state and divergence + verify Verify integrity of local and remote data`, + } + + cmd.AddCommand( + newVaultInitCmd(), + newVaultPushCmd(), + newVaultPullCmd(), + newVaultSyncCmd(), + newVaultStatusCmd(), + newVaultVerifyCmd(), + ) + + return cmd +} + +func newVaultInitCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + Short: "Configure vault remote and test connectivity", + RunE: func(cmd *cobra.Command, args []string) error { + jsonOut, _ := cmd.Flags().GetBool("json") + + uri, _ := cmd.Flags().GetString("uri") + endpoint, _ := cmd.Flags().GetString("endpoint") + region, _ := cmd.Flags().GetString("region") + accessKey, _ := cmd.Flags().GetString("access-key") + secretKey, _ := cmd.Flags().GetString("secret-key") + pathStyle, _ := cmd.Flags().GetBool("path-style") + machineID, _ := cmd.Flags().GetString("machine-id") + + if uri == "" { + return fmt.Errorf("--uri is required") + } + + // Load and update config + cfg, err := config.Load() + if err != nil { + cfg = config.Default() + } + + freshInit := !cfg.Vault.Configured() + + cfg.Vault.Remote.URI = uri + if endpoint != "" { + cfg.Vault.Remote.Endpoint = endpoint + } + if region != "" { + cfg.Vault.Remote.Region = region + } + if accessKey != "" { + cfg.Vault.Remote.AccessKeyID = accessKey + } else if cfg.Vault.Remote.AccessKeyID == "" { + cfg.Vault.Remote.AccessKeyID = os.Getenv("FLOOP_VAULT_ACCESS_KEY") + } + if secretKey != "" { + cfg.Vault.Remote.SecretAccessKey = secretKey + } else if cfg.Vault.Remote.SecretAccessKey == "" { + cfg.Vault.Remote.SecretAccessKey = os.Getenv("FLOOP_VAULT_SECRET_KEY") + } + cfg.Vault.Remote.PathStyle = pathStyle + if machineID != "" { + cfg.Vault.MachineID = machineID + } + + // Set defaults + if cfg.Vault.Remote.Region == "" { + cfg.Vault.Remote.Region = "us-east-1" + } + if cfg.Vault.Sync.Timeout == "" { + cfg.Vault.Sync.Timeout = "30s" + } + if freshInit { + cfg.Vault.Sync.IncludeProjects = true + } + + // Validate + if err := cfg.Vault.Validate(); err != nil { + return fmt.Errorf("invalid vault config: %w", err) + } + + // Test connectivity before saving — don't persist broken config + homeDir, homeErr := os.UserHomeDir() + if homeErr != nil { + return fmt.Errorf("cannot determine home directory: %w", homeErr) + } + vectorDir := filepath.Join(homeDir, ".floop", "vectors") + svc, err := vault.NewVaultService(&cfg.Vault, vectorDir, version, defaultVaultDims) + if err != nil { + return fmt.Errorf("creating vault service: %w", err) + } + + ctx := context.Background() + if err := svc.Init(ctx); err != nil { + return err + } + + // Save config only after successful connectivity test + if err := cfg.Save(); err != nil { + return fmt.Errorf("cannot save config: %w", err) + } + + resolvedMachineID := cfg.Vault.ResolveMachineID() + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ + "status": "initialized", + "uri": cfg.Vault.Remote.URI, + "endpoint": cfg.Vault.Remote.Endpoint, + "machine_id": resolvedMachineID, + "message": "Vault initialized. Run 'floop vault push' to sync.", + }) + } + + fmt.Printf("Vault initialized successfully.\n") + fmt.Printf(" URI: %s\n", cfg.Vault.Remote.URI) + fmt.Printf(" Endpoint: %s\n", cfg.Vault.Remote.Endpoint) + fmt.Printf(" Machine ID: %s\n", resolvedMachineID) + fmt.Printf("\nRun 'floop vault push' to sync.\n") + return nil + }, + } + + cmd.Flags().String("uri", "", "S3 URI (s3://bucket/prefix)") + cmd.Flags().String("endpoint", "", "S3 endpoint URL") + cmd.Flags().String("region", "us-east-1", "AWS region") + cmd.Flags().String("access-key", "", "Access key ID (or set FLOOP_VAULT_ACCESS_KEY)") + cmd.Flags().String("secret-key", "", "Secret access key — prefer FLOOP_VAULT_SECRET_KEY env var to avoid exposure in process listings") + cmd.Flags().Bool("path-style", true, "Use path-style requests (default true for MinIO)") + cmd.Flags().String("machine-id", "", "Machine identifier (default: hostname)") + + return cmd +} + +func newVaultPushCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "push", + Short: "Push local state to remote", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + jsonOut, _ := cmd.Flags().GetBool("json") + force, _ := cmd.Flags().GetBool("force") + dryRun, _ := cmd.Flags().GetBool("dry-run") + scope, _ := cmd.Flags().GetString("scope") + + svc, graphStore, cleanup, err := setupVaultCmd(root) + if err != nil { + return err + } + defer cleanup() + + ctx := context.Background() + result, err := svc.Push(ctx, graphStore, root, vault.PushOptions{ + Force: force, + DryRun: dryRun, + Scope: scope, + }) + if err != nil { + return fmt.Errorf("push failed: %w", err) + } + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ + "vector_rows_pushed": result.Vectors.RowsPushed, + "node_count": result.Graph.NodeCount, + "edge_count": result.Graph.EdgeCount, + "dry_run": dryRun, + "duration": result.Duration.String(), + "message": formatPushMessage(result, dryRun), + }) + } + + if dryRun { + fmt.Println("Would push:") + fmt.Printf(" Vectors: %d rows\n", result.Vectors.RowsPushed) + fmt.Printf(" Graph: %d nodes, %d edges\n", result.Graph.NodeCount, result.Graph.EdgeCount) + } else { + fmt.Printf("Push complete (%s)\n", result.Duration.Round(time.Millisecond)) + fmt.Printf(" Vectors: %d rows pushed\n", result.Vectors.RowsPushed) + fmt.Printf(" Graph: %d nodes, %d edges\n", result.Graph.NodeCount, result.Graph.EdgeCount) + } + return nil + }, + } + + cmd.Flags().Bool("force", false, "Overwrite remote state without diffing") + cmd.Flags().Bool("dry-run", false, "Show what would be pushed without pushing") + cmd.Flags().String("scope", "global", "Scope: global, local, or both") + + return cmd +} + +func newVaultPullCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "pull", + Short: "Pull remote state to local", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + jsonOut, _ := cmd.Flags().GetBool("json") + force, _ := cmd.Flags().GetBool("force") + dryRun, _ := cmd.Flags().GetBool("dry-run") + from, _ := cmd.Flags().GetString("from") + scope, _ := cmd.Flags().GetString("scope") + + svc, graphStore, cleanup, err := setupVaultCmd(root) + if err != nil { + return err + } + defer cleanup() + + ctx := context.Background() + result, err := svc.Pull(ctx, graphStore, vault.PullOptions{ + Force: force, + DryRun: dryRun, + FromMachine: from, + Scope: scope, + Root: root, + }) + if err != nil { + return fmt.Errorf("pull failed: %w", err) + } + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ + "vector_rows_pulled": result.Vectors.RowsPulled, + "node_count": result.Graph.NodeCount, + "edge_count": result.Graph.EdgeCount, + "dry_run": dryRun, + "duration": result.Duration.String(), + "message": formatPullMessage(result, dryRun), + }) + } + + if dryRun { + fmt.Println("Would pull:") + fmt.Printf(" Vectors: %d rows\n", result.Vectors.RowsPulled) + fmt.Printf(" Graph: %d nodes, %d edges\n", result.Graph.NodeCount, result.Graph.EdgeCount) + } else { + fmt.Printf("Pull complete (%s)\n", result.Duration.Round(time.Millisecond)) + fmt.Printf(" Vectors: %d rows pulled\n", result.Vectors.RowsPulled) + fmt.Printf(" Graph: %d nodes, %d edges\n", result.Graph.NodeCount, result.Graph.EdgeCount) + } + return nil + }, + } + + cmd.Flags().Bool("force", false, "Overwrite local state without diffing") + cmd.Flags().Bool("dry-run", false, "Show what would be pulled without pulling") + cmd.Flags().String("from", "", "Machine ID to pull from (default: own machine)") + cmd.Flags().String("scope", "global", "Scope: global, local, or both") + + return cmd +} + +func newVaultSyncCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "sync", + Short: "Bidirectional sync (pull then push)", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + jsonOut, _ := cmd.Flags().GetBool("json") + dryRun, _ := cmd.Flags().GetBool("dry-run") + scope, _ := cmd.Flags().GetString("scope") + + svc, graphStore, cleanup, err := setupVaultCmd(root) + if err != nil { + return err + } + defer cleanup() + + ctx := context.Background() + result, err := svc.Sync(ctx, graphStore, root, vault.SyncOptions{ + DryRun: dryRun, + Scope: scope, + }) + if err != nil { + return fmt.Errorf("sync failed: %w", err) + } + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{ + "pulled": map[string]interface{}{ + "vector_rows": result.Pulled.Vectors.RowsPulled, + "nodes": result.Pulled.Graph.NodeCount, + "edges": result.Pulled.Graph.EdgeCount, + }, + "pushed": map[string]interface{}{ + "vector_rows": result.Pushed.Vectors.RowsPushed, + "nodes": result.Pushed.Graph.NodeCount, + "edges": result.Pushed.Graph.EdgeCount, + }, + "message": "Sync complete", + }) + } + + fmt.Println("Sync complete") + fmt.Printf(" Pulled: %d vector rows, %d nodes, %d edges\n", + result.Pulled.Vectors.RowsPulled, result.Pulled.Graph.NodeCount, result.Pulled.Graph.EdgeCount) + fmt.Printf(" Pushed: %d vector rows, %d nodes, %d edges\n", + result.Pushed.Vectors.RowsPushed, result.Pushed.Graph.NodeCount, result.Pushed.Graph.EdgeCount) + return nil + }, + } + + cmd.Flags().Bool("dry-run", false, "Show sync plan without executing") + cmd.Flags().String("scope", "global", "Scope: global, local, or both") + + return cmd +} + +func newVaultStatusCmd() *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Show sync state and divergence", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + jsonOut, _ := cmd.Flags().GetBool("json") + + svc, graphStore, cleanup, err := setupVaultCmd(root) + if err != nil { + return err + } + defer cleanup() + + ctx := context.Background() + result, err := svc.Status(ctx, graphStore) + if err != nil { + return fmt.Errorf("status failed: %w", err) + } + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(result) + } + + fmt.Printf("Vault: configured (%s)\n", result.URI) + fmt.Printf("Machine: %s\n", result.MachineID) + if !result.LastPush.IsZero() { + fmt.Printf("Last push: %s (%s ago)\n", result.LastPush.Format(time.RFC3339), time.Since(result.LastPush).Round(time.Minute)) + } else { + fmt.Println("Last push: never") + } + if !result.LastPull.IsZero() { + fmt.Printf("Last pull: %s (%s ago)\n", result.LastPull.Format(time.RFC3339), time.Since(result.LastPull).Round(time.Minute)) + } else { + fmt.Println("Last pull: never") + } + fmt.Printf("Local: %d vector rows, %d nodes\n", result.LocalVectorRows, result.LocalNodeCount) + fmt.Printf("Remote: %d vector rows\n", result.RemoteVectorRows) + fmt.Printf("Status: %s\n", result.Status) + return nil + }, + } +} + +func newVaultVerifyCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "verify", + Short: "Verify integrity of local and remote data", + RunE: func(cmd *cobra.Command, args []string) error { + root, _ := cmd.Flags().GetString("root") + jsonOut, _ := cmd.Flags().GetBool("json") + remoteOnly, _ := cmd.Flags().GetBool("remote-only") + localOnly, _ := cmd.Flags().GetBool("local-only") + + svc, graphStore, cleanup, err := setupVaultCmd(root) + if err != nil { + return err + } + defer cleanup() + + ctx := context.Background() + result, err := svc.Verify(ctx, graphStore, vault.VerifyOptions{ + RemoteOnly: remoteOnly, + LocalOnly: localOnly, + }) + if err != nil { + return fmt.Errorf("verify failed: %w", err) + } + + if jsonOut { + return json.NewEncoder(os.Stdout).Encode(result) + } + + if result.OK { + fmt.Println("Verification passed") + } else { + fmt.Println("Verification FAILED") + } + for _, issue := range result.Issues { + fmt.Printf(" - %s\n", issue) + } + if result.LocalVectorRows >= 0 { + fmt.Printf(" Local vectors: %d rows\n", result.LocalVectorRows) + } + if result.RemoteVectorRows >= 0 { + fmt.Printf(" Remote vectors: %d rows\n", result.RemoteVectorRows) + } + if !result.OK { + return fmt.Errorf("verification failed with %d issue(s)", len(result.Issues)) + } + return nil + }, + } + + cmd.Flags().Bool("remote-only", false, "Only verify remote data") + cmd.Flags().Bool("local-only", false, "Only verify local data") + + return cmd +} + +// setupVaultCmd loads config, creates store and vault service. +func setupVaultCmd(root string) (*vault.VaultService, store.GraphStore, func(), error) { + cfg, err := config.Load() + if err != nil { + return nil, nil, nil, fmt.Errorf("loading config: %w", err) + } + + if !cfg.Vault.Configured() { + return nil, nil, nil, fmt.Errorf("vault not configured — run 'floop vault init'") + } + + graphStore, err := store.NewMultiGraphStore(root) + if err != nil { + return nil, nil, nil, fmt.Errorf("opening store: %w", err) + } + + homeDir, homeErr := os.UserHomeDir() + if homeErr != nil { + return nil, nil, nil, fmt.Errorf("cannot determine home directory: %w", homeErr) + } + vectorDir := filepath.Join(homeDir, ".floop", "vectors") + + svc, err := vault.NewVaultService(&cfg.Vault, vectorDir, version, defaultVaultDims) + if err != nil { + graphStore.Close() + return nil, nil, nil, err + } + + cleanup := func() { + graphStore.Close() + } + + return svc, graphStore, cleanup, nil +} + +func formatPushMessage(r *vault.PushResult, dryRun bool) string { + if dryRun { + return fmt.Sprintf("Would push: %d vectors, %d nodes, %d edges", + r.Vectors.RowsPushed, r.Graph.NodeCount, r.Graph.EdgeCount) + } + return fmt.Sprintf("Push complete: %d vectors, %d nodes, %d edges", + r.Vectors.RowsPushed, r.Graph.NodeCount, r.Graph.EdgeCount) +} + +func formatPullMessage(r *vault.PullResult, dryRun bool) string { + if dryRun { + return fmt.Sprintf("Would pull: %d vectors, %d nodes, %d edges", + r.Vectors.RowsPulled, r.Graph.NodeCount, r.Graph.EdgeCount) + } + return fmt.Sprintf("Pull complete: %d vectors, %d nodes, %d edges", + r.Vectors.RowsPulled, r.Graph.NodeCount, r.Graph.EdgeCount) +} diff --git a/cmd/floop/cmd_vault_test.go b/cmd/floop/cmd_vault_test.go new file mode 100644 index 00000000..7df10a05 --- /dev/null +++ b/cmd/floop/cmd_vault_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "testing" +) + +func TestVaultCmd_HasSubcommands(t *testing.T) { + cmd := newVaultCmd() + + subs := cmd.Commands() + names := make(map[string]bool) + for _, sub := range subs { + names[sub.Name()] = true + } + + expected := []string{"init", "push", "pull", "sync", "status", "verify"} + for _, name := range expected { + if !names[name] { + t.Errorf("missing subcommand: %s", name) + } + } +} + +func TestVaultInitCmd_RequiresURI(t *testing.T) { + cmd := newVaultInitCmd() + cmd.SetArgs([]string{}) + err := cmd.Execute() + if err == nil { + t.Fatal("expected error when --uri is missing") + } +} + +func TestVaultPushCmd_FlagParsing(t *testing.T) { + cmd := newVaultPushCmd() + // Just verify flags exist — we can't run RunE without config + f := cmd.Flags() + if f.Lookup("force") == nil { + t.Error("missing --force flag") + } + if f.Lookup("dry-run") == nil { + t.Error("missing --dry-run flag") + } + if f.Lookup("scope") == nil { + t.Error("missing --scope flag") + } +} + +func TestVaultPullCmd_FlagParsing(t *testing.T) { + cmd := newVaultPullCmd() + f := cmd.Flags() + if f.Lookup("force") == nil { + t.Error("missing --force flag") + } + if f.Lookup("from") == nil { + t.Error("missing --from flag") + } +} + +func TestVaultVerifyCmd_FlagParsing(t *testing.T) { + cmd := newVaultVerifyCmd() + f := cmd.Flags() + if f.Lookup("remote-only") == nil { + t.Error("missing --remote-only flag") + } + if f.Lookup("local-only") == nil { + t.Error("missing --local-only flag") + } +} diff --git a/cmd/floop/main.go b/cmd/floop/main.go index aa89a7e9..bcbd429f 100644 --- a/cmd/floop/main.go +++ b/cmd/floop/main.go @@ -177,6 +177,8 @@ context-aware behavior activation for consistent agent operation.`, // Backup/restore commands newBackupCmd(), newRestoreFromBackupCmd(), + // Vault sync commands + newVaultCmd(), // Hook management commands newUpgradeCmd(), // Tag management commands diff --git a/docker-compose.vault-test.yml b/docker-compose.vault-test.yml new file mode 100644 index 00000000..cb754e63 --- /dev/null +++ b/docker-compose.vault-test.yml @@ -0,0 +1,10 @@ +services: + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 2f83bcc3..bef40889 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -1364,6 +1364,133 @@ floop restore-backup backup.json.gz --json --- +## Vault Sync + +Commands for Lance-native S3 backup and synchronization of floop's behavioral memory store. + +### vault + +Parent command for vault sync operations. + +``` +floop vault [flags] +``` + +### vault init + +Configure vault remote and test connectivity. + +``` +floop vault init [flags] +``` + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--uri` | string | `""` | S3 URI (`s3://bucket/prefix`) — required | +| `--endpoint` | string | `""` | S3 endpoint URL (required for MinIO, R2) | +| `--region` | string | `"us-east-1"` | AWS region | +| `--access-key` | string | `""` | Access key ID (or set `FLOOP_VAULT_ACCESS_KEY`) | +| `--secret-key` | string | `""` | Secret access key (or set `FLOOP_VAULT_SECRET_KEY`) | +| `--path-style` | bool | `true` | Use path-style requests (MinIO default) | +| `--machine-id` | string | hostname | Machine identifier | + +**Examples:** + +```bash +# Initialize with MinIO on Tailnet +floop vault init --uri s3://floop-vault/brain \ + --endpoint https://minio.tailnet.ts.net:9000 \ + --access-key "$FLOOP_VAULT_ACCESS_KEY" \ + --secret-key "$FLOOP_VAULT_SECRET_KEY" + +# Initialize with JSON output +floop vault init --uri s3://floop-vault/brain --endpoint http://localhost:9000 --json +``` + +### vault push + +Push local state to remote. + +``` +floop vault push [flags] +``` + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--force` | bool | `false` | Overwrite remote state without diffing | +| `--dry-run` | bool | `false` | Show what would be pushed without pushing | +| `--scope` | string | `"global"` | Scope: `global`, `local`, or `both` | + +**Examples:** + +```bash +floop vault push +floop vault push --dry-run +floop vault push --scope both +``` + +### vault pull + +Pull remote state to local. + +``` +floop vault pull [flags] +``` + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--force` | bool | `false` | Overwrite local state without diffing | +| `--dry-run` | bool | `false` | Show what would be pulled without pulling | +| `--from` | string | own machine | Machine ID to pull from | +| `--scope` | string | `"global"` | Scope: `global`, `local`, or `both` | + +**Examples:** + +```bash +floop vault pull +floop vault pull --from laptop +floop vault pull --dry-run --json +``` + +### vault sync + +Bidirectional sync (pull first, then push). + +``` +floop vault sync [flags] +``` + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--dry-run` | bool | `false` | Show sync plan without executing | +| `--scope` | string | `"global"` | Scope: `global`, `local`, or `both` | + +**Examples:** + +```bash +floop vault sync +floop vault sync --dry-run +``` + +### vault status + +Show sync state and divergence between local and remote. + +``` +floop vault status +``` + +**Examples:** + +```bash +floop vault status +floop vault status --json +``` + +**See also:** [backup](#backup), [restore-backup](#restore-backup) + +--- + ## Hooks Commands called by Claude Code hooks for automatic behavior injection, correction detection, and dynamic context. These are native Go subcommands that replace the old shell script approach, enabling Windows support. diff --git a/go.mod b/go.mod index 1c3c0a3b..fa9eaa39 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -68,17 +69,24 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jupiterrider/ffi v0.6.0 // indirect github.com/klauspost/compress v1.18.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.100 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.6.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect @@ -91,6 +99,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/mod v0.33.0 // indirect @@ -114,4 +123,4 @@ require ( modernc.org/memory v1.11.0 // indirect ) -replace github.com/lancedb/lancedb-go => github.com/nvandessel/lancedb-go v0.2.1 +replace github.com/lancedb/lancedb-go => github.com/nvandessel/lancedb-go v0.2.2 diff --git a/go.sum b/go.sum index b5bc3644..cbac99a9 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,10 @@ cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KM cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -28,6 +32,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54= github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= @@ -70,13 +76,29 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8= +github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= @@ -91,6 +113,8 @@ github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -98,8 +122,12 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -140,37 +168,84 @@ github.com/jupiterrider/ffi v0.6.0 h1:UX378KcZvH5c8qgLi9KL/bL82SZTHdRspZ+jj7bvBn github.com/jupiterrider/ffi v0.6.0/go.mod h1:PqZ5Go6X9by8CIXgfprxfMPYmn8oT5m2O7AA56s64bY= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= +github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= +github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8= +github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modelcontextprotocol/go-sdk v1.5.0 h1:CHU0FIX9kpueNkxuYtfYQn1Z0slhFzBZuq+x6IiblIU= github.com/modelcontextprotocol/go-sdk v1.5.0/go.mod h1:gggDIhoemhWs3BGkGwd1umzEXCEMMvAnhTrnbXJKKKA= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/nvandessel/lancedb-go v0.2.1 h1:h+qHbg36rFojNMQZe3V6ZtoGH/HM9TNN6xI4VcLgLnw= -github.com/nvandessel/lancedb-go v0.2.1/go.mod h1:MIL9xwm6mYzZbFh23sYOJt17TU0fDAPeOvDE15/1P8k= +github.com/nvandessel/lancedb-go v0.2.2 h1:m+g219UubUvm6plcz4ZJ/SYykOUELT6/YE6J7EpDtjg= +github.com/nvandessel/lancedb-go v0.2.2/go.mod h1:BeMET0eOrNH2CyZMdaoCSY2uTCNI053Aq2lFLfoxVuc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= @@ -179,10 +254,22 @@ github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMps github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= +github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= +github.com/testcontainers/testcontainers-go/modules/minio v0.33.0 h1:lHhjYlm0Oh+PfM03NIwCqNg2zSz9VuNTwUKi4MQfYAA= +github.com/testcontainers/testcontainers-go/modules/minio v0.33.0/go.mod h1:3WRFF6lLI3IqXb7lvOx6OpEcH1jgs59mbzZiPTJeEJg= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -207,6 +294,7 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= @@ -220,7 +308,6 @@ golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/internal/config/config.go b/internal/config/config.go index b9af396b..8ecc24d1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ import ( "github.com/nvandessel/floop/internal/constants" "github.com/nvandessel/floop/internal/utils" + "github.com/nvandessel/floop/internal/vault" "gopkg.in/yaml.v3" ) @@ -40,6 +41,9 @@ type FloopConfig struct { // Events contains settings for the raw event buffer. Events EventsConfig `json:"events" yaml:"events"` + + // Vault contains settings for vault sync (Lance-native S3 backup). + Vault vault.VaultConfig `json:"vault" yaml:"vault"` } // TokenBudgetConfig configures token budget limits for behavior injection. @@ -281,6 +285,9 @@ func LoadFromFile(path string) (*FloopConfig, error) { // Expand environment variables in API key config.LLM.APIKey = expandEnvVars(config.LLM.APIKey) + // Expand environment variables in vault credentials + config.Vault.ExpandEnvVars() + return config, nil } @@ -339,6 +346,11 @@ func (c *FloopConfig) Validate() error { return fmt.Errorf("events.retention_days must be non-negative, got %d", c.Events.RetentionDays) } + // Vault validation + if err := c.Vault.Validate(); err != nil { + return err + } + return nil } diff --git a/internal/spreading/sproink_cgo.go b/internal/spreading/sproink_cgo.go index b6c4de73..d2c08053 100644 --- a/internal/spreading/sproink_cgo.go +++ b/internal/spreading/sproink_cgo.go @@ -16,6 +16,7 @@ import "C" import ( "context" "fmt" + "math" "sort" "sync" "sync/atomic" @@ -56,38 +57,54 @@ func sproinkGraphFree(graph *C.SproinkGraph) { } // sproinkActivate runs spreading activation on the graph. +// +// seedSources may be nil (all seeds map to NodeId::None on the engine side). +// temporalDecayRate and currentTime use NaN as the "not set" sentinel. func sproinkActivate( graph *C.SproinkGraph, seedNodes []uint32, seedActivations []float64, + seedSources []uint32, maxSteps uint32, decayFactor, spreadFactor, minActivation float64, sigmoidGain, sigmoidCenter float64, inhibitionEnabled bool, inhibitionStrength float64, inhibitionBreadth uint32, + temporalDecayRate, currentTime float64, ) (*C.SproinkResults, error) { numSeeds := len(seedNodes) var snPtr *C.uint32_t var saPtr *C.double + var ssPtr *C.uint32_t if numSeeds > 0 { snPtr = (*C.uint32_t)(unsafe.Pointer(&seedNodes[0])) saPtr = (*C.double)(unsafe.Pointer(&seedActivations[0])) + if len(seedSources) > 0 { + ssPtr = (*C.uint32_t)(unsafe.Pointer(&seedSources[0])) + } + } + + var inh C.uint8_t + if inhibitionEnabled { + inh = 1 } results := C.sproink_activate( graph, C.uint32_t(numSeeds), - snPtr, saPtr, + snPtr, saPtr, ssPtr, C.uint32_t(maxSteps), C.double(decayFactor), C.double(spreadFactor), C.double(minActivation), C.double(sigmoidGain), C.double(sigmoidCenter), - C.bool(inhibitionEnabled), + inh, C.double(inhibitionStrength), C.uint32_t(inhibitionBreadth), + C.double(temporalDecayRate), + C.double(currentTime), ) if results == nil { return nil, fmt.Errorf("sproink_activate returned nil") @@ -107,7 +124,7 @@ func sproinkResultsNodes(results *C.SproinkResults) []uint32 { return nil } nodes := make([]uint32, n) - C.sproink_results_nodes(results, (*C.uint32_t)(unsafe.Pointer(&nodes[0]))) + C.sproink_results_nodes(results, (*C.uint32_t)(unsafe.Pointer(&nodes[0])), C.uint32_t(n)) return nodes } @@ -118,7 +135,7 @@ func sproinkResultsActivations(results *C.SproinkResults) []float64 { return nil } activations := make([]float64, n) - C.sproink_results_activations(results, (*C.double)(unsafe.Pointer(&activations[0]))) + C.sproink_results_activations(results, (*C.double)(unsafe.Pointer(&activations[0])), C.uint32_t(n)) return activations } @@ -129,7 +146,7 @@ func sproinkResultsDistances(results *C.SproinkResults) []uint32 { return nil } distances := make([]uint32, n) - C.sproink_results_distances(results, (*C.uint32_t)(unsafe.Pointer(&distances[0]))) + C.sproink_results_distances(results, (*C.uint32_t)(unsafe.Pointer(&distances[0])), C.uint32_t(n)) return distances } @@ -181,6 +198,7 @@ func sproinkPairsData(pairs *C.SproinkPairs) (nodesA, nodesB []uint32, activatio pairs, (*C.uint32_t)(unsafe.Pointer(&nodesA[0])), (*C.uint32_t)(unsafe.Pointer(&nodesB[0])), + C.uint32_t(n), ) activationsA = make([]float64, n) @@ -189,6 +207,7 @@ func sproinkPairsData(pairs *C.SproinkPairs) (nodesA, nodesB []uint32, activatio pairs, (*C.double)(unsafe.Pointer(&activationsA[0])), (*C.double)(unsafe.Pointer(&activationsB[0])), + C.uint32_t(n), ) return nodesA, nodesB, activationsA, activationsB @@ -290,6 +309,7 @@ func (e *NativeEngine) Activate(ctx context.Context, seeds []Seed) ([]Result, er e.graph, seedNodes, seedActivations, + nil, // seedSources: floop performs string-based attribution post-hoc uint32(e.config.MaxSteps), e.config.DecayFactor, e.config.SpreadFactor, @@ -299,6 +319,8 @@ func (e *NativeEngine) Activate(ctx context.Context, seeds []Seed) ([]Result, er inhEnabled, inhStrength, inhBreadth, + math.NaN(), // temporalDecayRate: not set + math.NaN(), // currentTime: not set ) if err != nil { return nil, fmt.Errorf("NativeEngine.Activate: %w", err) diff --git a/internal/spreading/sproink_pairs_cgo.go b/internal/spreading/sproink_pairs_cgo.go index 5cdbfbfc..349f34c1 100644 --- a/internal/spreading/sproink_pairs_cgo.go +++ b/internal/spreading/sproink_pairs_cgo.go @@ -8,7 +8,11 @@ package spreading */ import "C" -import "github.com/nvandessel/floop/internal/constants" +import ( + "math" + + "github.com/nvandessel/floop/internal/constants" +) // NativeExtractPairs extracts co-activation pairs using sproink's FFI. // It maps u32 node IDs back to UUID strings via the engine's IDMap and @@ -91,6 +95,7 @@ func (e *NativeEngine) nativeActivateAndExtractPairs(seeds []Seed, threshold flo e.graph, seedNodes, seedActivations, + nil, uint32(e.config.MaxSteps), e.config.DecayFactor, e.config.SpreadFactor, @@ -100,6 +105,8 @@ func (e *NativeEngine) nativeActivateAndExtractPairs(seeds []Seed, threshold flo inhEnabled, inhStrength, inhBreadth, + math.NaN(), + math.NaN(), ) if err != nil { return nil, nil, err diff --git a/internal/vault/config.go b/internal/vault/config.go new file mode 100644 index 00000000..5a40b1c1 --- /dev/null +++ b/internal/vault/config.go @@ -0,0 +1,213 @@ +// Package vault implements Lance-native S3 backup and sync for floop's +// behavioral memory store. +package vault + +import ( + "fmt" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/lancedb/lancedb-go/pkg/contracts" +) + +// VaultConfig configures vault sync. +type VaultConfig struct { + Remote VaultRemoteConfig `json:"remote" yaml:"remote"` + MachineID string `json:"machine_id" yaml:"machine_id"` + Sync VaultSyncConfig `json:"sync" yaml:"sync"` + Encryption VaultEncryptionConfig `json:"encryption" yaml:"encryption"` +} + +// VaultRemoteConfig configures the S3-compatible remote endpoint. +type VaultRemoteConfig struct { + URI string `json:"uri" yaml:"uri"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + Region string `json:"region" yaml:"region"` + AccessKeyID string `json:"access_key_id" yaml:"access_key_id"` + SecretAccessKey string `json:"secret_access_key" yaml:"secret_access_key"` + PathStyle bool `json:"path_style" yaml:"path_style"` + AllowHTTP bool `json:"allow_http" yaml:"allow_http"` +} + +// VaultSyncConfig configures sync behavior. +type VaultSyncConfig struct { + AutoPush bool `json:"auto_push" yaml:"auto_push"` + IncludeProjects bool `json:"include_projects" yaml:"include_projects"` + Timeout string `json:"timeout" yaml:"timeout"` +} + +// VaultEncryptionConfig configures client-side encryption with age. +type VaultEncryptionConfig struct { + Enabled bool `json:"enabled" yaml:"enabled"` + IdentityFile string `json:"identity_file" yaml:"identity_file"` + Recipient string `json:"recipient" yaml:"recipient"` +} + +// machineIDRegex validates machine IDs as safe path components. +var machineIDRegex = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + +// Configured returns true if the vault remote is configured (URI is set). +func (c *VaultConfig) Configured() bool { + return c.Remote.URI != "" +} + +// Validate checks that the vault configuration is valid. +// Only validates if the vault is configured (URI is set). +func (c *VaultConfig) Validate() error { + if !c.Configured() { + return nil // unconfigured vault is valid (just unused) + } + + // Validate remote + if !strings.HasPrefix(c.Remote.URI, "s3://") { + return fmt.Errorf("vault.remote.uri: must start with s3://, got %q", c.Remote.URI) + } + bucket, _ := parseS3URI(c.Remote.URI) + if bucket == "" { + return fmt.Errorf("vault.remote.uri: must contain a bucket name") + } + + if c.Remote.Endpoint == "" { + return fmt.Errorf("vault.remote.endpoint: required (use --endpoint or vault.remote.endpoint in config)") + } + if _, err := url.Parse(c.Remote.Endpoint); err != nil { + return fmt.Errorf("vault.remote.endpoint: invalid URL: %w", err) + } + + if c.Remote.Region == "" { + return fmt.Errorf("vault.remote.region: required") + } + + if c.Remote.AccessKeyID == "" { + return fmt.Errorf("vault.remote.access_key_id: required") + } + + if c.Remote.SecretAccessKey == "" { + return fmt.Errorf("vault.remote.secret_access_key: required") + } + + // Validate machine_id + machineID := c.ResolveMachineID() + if !machineIDRegex.MatchString(machineID) { + return fmt.Errorf("vault.machine_id: must be alphanumeric with hyphens, underscores, or dots; got %q", machineID) + } + + // Validate sync timeout + if c.Sync.Timeout != "" { + d, err := time.ParseDuration(c.Sync.Timeout) + if err != nil { + return fmt.Errorf("vault.sync.timeout: invalid duration: %w", err) + } + if d > 10*time.Minute { + return fmt.Errorf("vault.sync.timeout: must be <= 10m, got %s", c.Sync.Timeout) + } + } + + // Validate encryption + if c.Encryption.Enabled { + if c.Encryption.IdentityFile == "" { + return fmt.Errorf("vault.encryption.identity_file: required when encryption is enabled") + } + if _, err := os.Stat(c.Encryption.IdentityFile); err != nil { + return fmt.Errorf("vault.encryption.identity_file: %w", err) + } + if c.Encryption.Recipient == "" { + return fmt.Errorf("vault.encryption.recipient: required when encryption is enabled") + } + } + + return nil +} + +// ResolveMachineID returns the effective machine ID. +// Falls back to os.Hostname(), then "localhost". +func (c *VaultConfig) ResolveMachineID() string { + if c.MachineID != "" { + return c.MachineID + } + hostname, err := os.Hostname() + if err != nil || hostname == "" { + return "localhost" + } + return hostname +} + +// SyncTimeout returns the parsed sync timeout duration, defaulting to 30s. +func (c *VaultConfig) SyncTimeout() time.Duration { + if c.Sync.Timeout == "" { + return 30 * time.Second + } + d, err := time.ParseDuration(c.Sync.Timeout) + if err != nil { + return 30 * time.Second + } + return d +} + +// StorageOptions builds the lancedb-go ConnectionOptions storage map. +func (c *VaultRemoteConfig) StorageOptions() map[string]string { + opts := map[string]string{ + contracts.StorageAccessKeyID: c.AccessKeyID, + contracts.StorageSecretAccessKey: c.SecretAccessKey, + contracts.StorageRegion: c.Region, + contracts.StorageVirtualHostedStyleRequest: strconv.FormatBool(!c.PathStyle), + contracts.StorageAllowHTTP: strconv.FormatBool(c.AllowHTTP), + } + if c.Endpoint != "" { + opts[contracts.StorageEndpoint] = c.Endpoint + } + return opts +} + +// RedactedSecretKey returns the secret key with most characters masked. +func (c *VaultRemoteConfig) RedactedSecretKey() string { + if c.SecretAccessKey == "" { + return "" + } + if len(c.SecretAccessKey) < 12 { + return "(set)" + } + return c.SecretAccessKey[:4] + "..." + c.SecretAccessKey[len(c.SecretAccessKey)-4:] +} + +// String implements fmt.Stringer with secret redaction. +func (c VaultRemoteConfig) String() string { + return fmt.Sprintf("VaultRemoteConfig{URI:%s, Endpoint:%s, Region:%s, AccessKeyID:%s, SecretAccessKey:%s}", + c.URI, c.Endpoint, c.Region, c.AccessKeyID, c.RedactedSecretKey()) +} + +// parseS3URI extracts bucket and prefix from an s3:// URI. +// Returns bucket="" if the URI is invalid. +func parseS3URI(uri string) (bucket, prefix string) { + if !strings.HasPrefix(uri, "s3://") { + return "", "" + } + rest := strings.TrimPrefix(uri, "s3://") + if rest == "" { + return "", "" + } + parts := strings.SplitN(rest, "/", 2) + bucket = parts[0] + if len(parts) > 1 { + prefix = parts[1] + } + return bucket, prefix +} + +// ExpandEnvVars expands ${VAR} patterns in vault credential fields. +func (c *VaultConfig) ExpandEnvVars() { + c.Remote.AccessKeyID = expandEnvVars(c.Remote.AccessKeyID) + c.Remote.SecretAccessKey = expandEnvVars(c.Remote.SecretAccessKey) +} + +// expandEnvVars expands ${VAR} patterns in a string. +func expandEnvVars(s string) string { + if !strings.Contains(s, "${") { + return s + } + return os.Expand(s, os.Getenv) +} diff --git a/internal/vault/config_test.go b/internal/vault/config_test.go new file mode 100644 index 00000000..bc2ed33d --- /dev/null +++ b/internal/vault/config_test.go @@ -0,0 +1,369 @@ +package vault + +import ( + "os" + "path/filepath" + "testing" + + "github.com/lancedb/lancedb-go/pkg/contracts" +) + +func TestVaultConfig_Validate(t *testing.T) { + validConfig := func() VaultConfig { + return VaultConfig{ + Remote: VaultRemoteConfig{ + URI: "s3://floop-vault/brain", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "AKIAIOSFODNN7EXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + PathStyle: true, + }, + MachineID: "workstation", + Sync: VaultSyncConfig{ + Timeout: "30s", + IncludeProjects: true, + }, + } + } + + t.Run("valid config passes", func(t *testing.T) { + cfg := validConfig() + if err := cfg.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("unconfigured vault is valid", func(t *testing.T) { + cfg := VaultConfig{} + if err := cfg.Validate(); err != nil { + t.Fatalf("empty config should be valid: %v", err) + } + }) + + tests := []struct { + name string + modify func(*VaultConfig) + errStr string + }{ + { + name: "invalid URI scheme", + modify: func(c *VaultConfig) { c.Remote.URI = "http://bucket/prefix" }, + errStr: "must start with s3://", + }, + { + name: "URI without bucket", + modify: func(c *VaultConfig) { c.Remote.URI = "s3://" }, + errStr: "must contain a bucket name", + }, + { + name: "empty endpoint", + modify: func(c *VaultConfig) { c.Remote.Endpoint = "" }, + errStr: "vault.remote.endpoint: required", + }, + { + name: "empty region", + modify: func(c *VaultConfig) { c.Remote.Region = "" }, + errStr: "vault.remote.region: required", + }, + { + name: "empty access key", + modify: func(c *VaultConfig) { c.Remote.AccessKeyID = "" }, + errStr: "vault.remote.access_key_id: required", + }, + { + name: "empty secret key", + modify: func(c *VaultConfig) { c.Remote.SecretAccessKey = "" }, + errStr: "vault.remote.secret_access_key: required", + }, + { + name: "invalid machine_id with slash", + modify: func(c *VaultConfig) { c.MachineID = "bad/id" }, + errStr: "must be alphanumeric", + }, + { + name: "invalid machine_id with space", + modify: func(c *VaultConfig) { c.MachineID = "bad id" }, + errStr: "must be alphanumeric", + }, + { + name: "invalid timeout duration", + modify: func(c *VaultConfig) { c.Sync.Timeout = "not-a-duration" }, + errStr: "invalid duration", + }, + { + name: "timeout exceeds max", + modify: func(c *VaultConfig) { c.Sync.Timeout = "15m" }, + errStr: "must be <= 10m", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + tt.modify(&cfg) + err := cfg.Validate() + if err == nil { + t.Fatal("expected error, got nil") + } + if got := err.Error(); !contains(got, tt.errStr) { + t.Errorf("error %q should contain %q", got, tt.errStr) + } + }) + } +} + +func TestVaultConfig_ValidMachineIDs(t *testing.T) { + validIDs := []string{"my-host", "my.host.fqdn", "host_1", "UPPER", "host123"} + for _, id := range validIDs { + t.Run(id, func(t *testing.T) { + if !machineIDRegex.MatchString(id) { + t.Errorf("machine ID %q should be valid", id) + } + }) + } +} + +func TestVaultConfig_EncryptionValidation(t *testing.T) { + t.Run("encryption enabled without identity file", func(t *testing.T) { + cfg := VaultConfig{ + Remote: VaultRemoteConfig{ + URI: "s3://bucket/prefix", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + }, + MachineID: "test", + Encryption: VaultEncryptionConfig{ + Enabled: true, + Recipient: "age1xyz...", + }, + } + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing identity file") + } + if !contains(err.Error(), "identity_file") { + t.Errorf("expected identity_file error, got: %v", err) + } + }) + + t.Run("encryption enabled with nonexistent identity file", func(t *testing.T) { + cfg := VaultConfig{ + Remote: VaultRemoteConfig{ + URI: "s3://bucket/prefix", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + }, + MachineID: "test", + Encryption: VaultEncryptionConfig{ + Enabled: true, + IdentityFile: "/nonexistent/key.txt", + Recipient: "age1xyz...", + }, + } + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for nonexistent identity file") + } + }) + + t.Run("encryption enabled without recipient", func(t *testing.T) { + tmp := t.TempDir() + keyFile := filepath.Join(tmp, "key.txt") + os.WriteFile(keyFile, []byte("key"), 0600) + + cfg := VaultConfig{ + Remote: VaultRemoteConfig{ + URI: "s3://bucket/prefix", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + }, + MachineID: "test", + Encryption: VaultEncryptionConfig{ + Enabled: true, + IdentityFile: keyFile, + }, + } + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing recipient") + } + if !contains(err.Error(), "recipient") { + t.Errorf("expected recipient error, got: %v", err) + } + }) +} + +func TestVaultRemoteConfig_StorageOptions(t *testing.T) { + cfg := VaultRemoteConfig{ + URI: "s3://floop-vault/brain", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "AKID", + SecretAccessKey: "SECRET", + PathStyle: true, + AllowHTTP: false, + } + + opts := cfg.StorageOptions() + + expected := map[string]string{ + contracts.StorageAccessKeyID: "AKID", + contracts.StorageSecretAccessKey: "SECRET", + contracts.StorageRegion: "us-east-1", + contracts.StorageEndpoint: "https://minio.example.com:9000", + contracts.StorageVirtualHostedStyleRequest: "false", // PathStyle=true → VirtualHosted=false + contracts.StorageAllowHTTP: "false", + } + + for k, want := range expected { + if got := opts[k]; got != want { + t.Errorf("StorageOptions[%q] = %q, want %q", k, got, want) + } + } +} + +func TestVaultRemoteConfig_StorageOptions_NoEndpoint(t *testing.T) { + cfg := VaultRemoteConfig{ + URI: "s3://bucket", + Region: "eu-west-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + } + opts := cfg.StorageOptions() + if _, ok := opts[contracts.StorageEndpoint]; ok { + t.Error("endpoint should not be set when VaultRemoteConfig.Endpoint is empty") + } +} + +func TestVaultRemoteConfig_String_RedactsSecret(t *testing.T) { + cfg := VaultRemoteConfig{ + URI: "s3://bucket/prefix", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + } + s := cfg.String() + if contains(s, "wJalrXUtnFEMI") { + t.Errorf("String() should not contain the raw secret key: %s", s) + } + if !contains(s, "wJal") { + t.Errorf("String() should show first 4 chars of secret: %s", s) + } +} + +func TestVaultRemoteConfig_RedactedSecretKey(t *testing.T) { + tests := []struct { + name string + key string + want string + }{ + {"empty", "", ""}, + {"short", "abc", "(set)"}, + {"normal", "wJalrXUtnFEMI/K7MDENG", "wJal...DENG"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := VaultRemoteConfig{SecretAccessKey: tt.key} + got := cfg.RedactedSecretKey() + if got != tt.want { + t.Errorf("RedactedSecretKey() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestParseS3URI(t *testing.T) { + tests := []struct { + uri string + wantBucket string + wantPrefix string + }{ + {"s3://bucket", "bucket", ""}, + {"s3://bucket/prefix", "bucket", "prefix"}, + {"s3://bucket/deep/prefix", "bucket", "deep/prefix"}, + {"s3://", "", ""}, + {"http://bucket", "", ""}, + } + for _, tt := range tests { + t.Run(tt.uri, func(t *testing.T) { + bucket, prefix := parseS3URI(tt.uri) + if bucket != tt.wantBucket { + t.Errorf("bucket = %q, want %q", bucket, tt.wantBucket) + } + if prefix != tt.wantPrefix { + t.Errorf("prefix = %q, want %q", prefix, tt.wantPrefix) + } + }) + } +} + +func TestVaultConfig_ExpandEnvVars(t *testing.T) { + t.Setenv("TEST_VAULT_KEY", "my-access-key") + t.Setenv("TEST_VAULT_SECRET", "my-secret-key") + + cfg := VaultConfig{ + Remote: VaultRemoteConfig{ + AccessKeyID: "${TEST_VAULT_KEY}", + SecretAccessKey: "${TEST_VAULT_SECRET}", + }, + } + cfg.ExpandEnvVars() + + if cfg.Remote.AccessKeyID != "my-access-key" { + t.Errorf("AccessKeyID = %q, want %q", cfg.Remote.AccessKeyID, "my-access-key") + } + if cfg.Remote.SecretAccessKey != "my-secret-key" { + t.Errorf("SecretAccessKey = %q, want %q", cfg.Remote.SecretAccessKey, "my-secret-key") + } +} + +func TestVaultConfig_ResolveMachineID(t *testing.T) { + t.Run("explicit ID", func(t *testing.T) { + cfg := VaultConfig{MachineID: "my-machine"} + if got := cfg.ResolveMachineID(); got != "my-machine" { + t.Errorf("got %q, want %q", got, "my-machine") + } + }) + + t.Run("falls back to hostname", func(t *testing.T) { + cfg := VaultConfig{} + got := cfg.ResolveMachineID() + if got == "" { + t.Error("ResolveMachineID should not return empty string") + } + }) +} + +func TestVaultConfig_SyncTimeout(t *testing.T) { + t.Run("default", func(t *testing.T) { + cfg := VaultConfig{} + if got := cfg.SyncTimeout(); got.String() != "30s" { + t.Errorf("default timeout = %s, want 30s", got) + } + }) + + t.Run("custom", func(t *testing.T) { + cfg := VaultConfig{Sync: VaultSyncConfig{Timeout: "1m"}} + if got := cfg.SyncTimeout(); got.String() != "1m0s" { + t.Errorf("timeout = %s, want 1m0s", got) + } + }) +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && containsStr(s, substr) +} + +func containsStr(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/internal/vault/encrypt.go b/internal/vault/encrypt.go new file mode 100644 index 00000000..0928d4c1 --- /dev/null +++ b/internal/vault/encrypt.go @@ -0,0 +1,39 @@ +package vault + +import ( + "fmt" + "os" + "os/exec" +) + +// EncryptFile encrypts src to dst using the given age recipient (public key). +func EncryptFile(recipient string, src, dst string) error { + agePath, err := exec.LookPath("age") + if err != nil { + return fmt.Errorf("age binary not found: install from https://age-encryption.org/") + } + + cmd := exec.Command(agePath, "-r", recipient, "-o", dst, src) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + os.Remove(dst) // clean up on failure + return fmt.Errorf("encrypting %s: %w", src, err) + } + return nil +} + +// DecryptFile decrypts src to dst using the given age identity file. +func DecryptFile(identityFile string, src, dst string) error { + agePath, err := exec.LookPath("age") + if err != nil { + return fmt.Errorf("age binary not found: install from https://age-encryption.org/") + } + + cmd := exec.Command(agePath, "-d", "-i", identityFile, "-o", dst, src) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + os.Remove(dst) // clean up on failure + return fmt.Errorf("decrypting %s: %w", src, err) + } + return nil +} diff --git a/internal/vault/encrypt_test.go b/internal/vault/encrypt_test.go new file mode 100644 index 00000000..39412de8 --- /dev/null +++ b/internal/vault/encrypt_test.go @@ -0,0 +1,136 @@ +package vault + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestEncryptDecrypt_RoundTrip(t *testing.T) { + if _, err := exec.LookPath("age"); err != nil { + t.Skip("age not installed") + } + if _, err := exec.LookPath("age-keygen"); err != nil { + t.Skip("age-keygen not installed") + } + + dir := t.TempDir() + + // Generate keypair + keygenOut, err := exec.Command("age-keygen").Output() + if err != nil { + t.Fatalf("age-keygen failed: %v", err) + } + + // Parse recipient from output (line starting with "public key:") + var recipient string + for _, line := range strings.Split(string(keygenOut), "\n") { + if strings.HasPrefix(line, "# public key: ") { + recipient = strings.TrimPrefix(line, "# public key: ") + break + } + } + if recipient == "" { + t.Fatal("could not parse public key from age-keygen output") + } + + // Write identity file + identityFile := filepath.Join(dir, "key.txt") + if err := os.WriteFile(identityFile, keygenOut, 0600); err != nil { + t.Fatalf("writing identity file: %v", err) + } + + // Write test file + srcFile := filepath.Join(dir, "test.txt") + want := "hello vault encryption" + if err := os.WriteFile(srcFile, []byte(want), 0600); err != nil { + t.Fatalf("writing test file: %v", err) + } + + // Encrypt + encFile := filepath.Join(dir, "test.txt.age") + if err := EncryptFile(recipient, srcFile, encFile); err != nil { + t.Fatalf("EncryptFile: %v", err) + } + + // Verify encrypted file exists and differs from original + encData, err := os.ReadFile(encFile) + if err != nil { + t.Fatalf("reading encrypted file: %v", err) + } + if string(encData) == want { + t.Error("encrypted file should differ from plaintext") + } + + // Decrypt + decFile := filepath.Join(dir, "test.decrypted.txt") + if err := DecryptFile(identityFile, encFile, decFile); err != nil { + t.Fatalf("DecryptFile: %v", err) + } + + got, err := os.ReadFile(decFile) + if err != nil { + t.Fatalf("reading decrypted file: %v", err) + } + if string(got) != want { + t.Errorf("decrypted content = %q, want %q", string(got), want) + } +} + +func TestEncryptFile_BadRecipient(t *testing.T) { + if _, err := exec.LookPath("age"); err != nil { + t.Skip("age not installed") + } + + dir := t.TempDir() + src := filepath.Join(dir, "test.txt") + os.WriteFile(src, []byte("test"), 0600) + dst := filepath.Join(dir, "test.age") + + err := EncryptFile("not-a-valid-recipient", src, dst) + if err == nil { + t.Fatal("expected error for bad recipient") + } +} + +func TestDecryptFile_WrongKey(t *testing.T) { + if _, err := exec.LookPath("age"); err != nil { + t.Skip("age not installed") + } + if _, err := exec.LookPath("age-keygen"); err != nil { + t.Skip("age-keygen not installed") + } + + dir := t.TempDir() + + // Generate two keypairs + keygen1, _ := exec.Command("age-keygen").Output() + keygen2, _ := exec.Command("age-keygen").Output() + + var recipient1 string + for _, line := range strings.Split(string(keygen1), "\n") { + if strings.HasPrefix(line, "# public key: ") { + recipient1 = strings.TrimPrefix(line, "# public key: ") + break + } + } + + // Write identity file for key 2 + idFile2 := filepath.Join(dir, "key2.txt") + os.WriteFile(idFile2, keygen2, 0600) + + // Encrypt with key 1 + src := filepath.Join(dir, "test.txt") + os.WriteFile(src, []byte("secret"), 0600) + enc := filepath.Join(dir, "test.age") + EncryptFile(recipient1, src, enc) + + // Try to decrypt with key 2 + dec := filepath.Join(dir, "test.dec") + err := DecryptFile(idFile2, enc, dec) + if err == nil { + t.Fatal("expected error when decrypting with wrong key") + } +} diff --git a/internal/vault/graph_sync.go b/internal/vault/graph_sync.go new file mode 100644 index 00000000..49aa5b32 --- /dev/null +++ b/internal/vault/graph_sync.go @@ -0,0 +1,267 @@ +package vault + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/nvandessel/floop/internal/backup" + "github.com/nvandessel/floop/internal/store" +) + +// GraphSyncer syncs graph data (V2 backup + corrections.jsonl) via S3. +type GraphSyncer struct { + s3 S3Operations + machineID string + encryption *VaultEncryptionConfig +} + +// GraphSyncResult contains the results of a graph sync operation. +type GraphSyncResult struct { + NodeCount int + EdgeCount int + CorrectionsSize int64 +} + +// NewGraphSyncer creates a GraphSyncer. +func NewGraphSyncer(s3 S3Operations, machineID string, enc *VaultEncryptionConfig) *GraphSyncer { + gs := &GraphSyncer{ + s3: s3, + machineID: machineID, + } + if enc != nil && enc.Enabled { + gs.encryption = enc + } + return gs +} + +// Push exports the graph as a V2 backup and uploads it along with corrections.jsonl. +func (g *GraphSyncer) Push(ctx context.Context, graphStore store.GraphStore, correctionsPath string, floopVersion string) (*GraphSyncResult, error) { + result := &GraphSyncResult{} + + // Export V2 backup + tmpDir, err := os.MkdirTemp("", "floop-vault-push-*") + if err != nil { + return nil, fmt.Errorf("creating temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + backupPath := filepath.Join(tmpDir, "floop-backup.json.gz") + bf, err := backup.BackupWithOptions(ctx, graphStore, backupPath, backup.BackupOptions{ + Compress: true, + FloopVersion: floopVersion, + }) + if err != nil { + return nil, fmt.Errorf("creating backup: %w", err) + } + result.NodeCount = len(bf.Nodes) + result.EdgeCount = len(bf.Edges) + + uploadPath := backupPath + if g.encryption != nil { + encPath := backupPath + ".age" + if err := EncryptFile(g.encryption.Recipient, backupPath, encPath); err != nil { + return nil, fmt.Errorf("encrypting backup: %w", err) + } + uploadPath = encPath + } + + // Upload backup + backupKey := fmt.Sprintf("machines/%s/graph/floop-backup.json.gz", g.machineID) + if err := g.uploadFile(ctx, backupKey, uploadPath); err != nil { + return nil, fmt.Errorf("uploading backup: %w", err) + } + + // Upload corrections.jsonl if it exists + if correctionsPath != "" { + if info, statErr := os.Stat(correctionsPath); statErr == nil { + result.CorrectionsSize = info.Size() + + corrUploadPath := correctionsPath + if g.encryption != nil { + encCorrPath := filepath.Join(tmpDir, "corrections.jsonl.age") + if err := EncryptFile(g.encryption.Recipient, correctionsPath, encCorrPath); err != nil { + return nil, fmt.Errorf("encrypting corrections: %w", err) + } + corrUploadPath = encCorrPath + } + + corrKey := fmt.Sprintf("machines/%s/graph/corrections.jsonl", g.machineID) + if err := g.uploadFile(ctx, corrKey, corrUploadPath); err != nil { + return nil, fmt.Errorf("uploading corrections: %w", err) + } + } + } + + return result, nil +} + +// Pull downloads the V2 backup and corrections from another machine and restores them. +func (g *GraphSyncer) Pull(ctx context.Context, graphStore store.GraphStore, fromMachineID string, correctionsPath string) (*GraphSyncResult, error) { + result := &GraphSyncResult{} + + tmpDir, err := os.MkdirTemp("", "floop-vault-pull-*") + if err != nil { + return nil, fmt.Errorf("creating temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + // Download backup + backupKey := fmt.Sprintf("machines/%s/graph/floop-backup.json.gz", fromMachineID) + downloadPath := filepath.Join(tmpDir, "floop-backup.json.gz") + + if err := g.downloadFile(ctx, backupKey, downloadPath); err != nil { + return nil, fmt.Errorf("downloading backup: %w", err) + } + + restorePath := downloadPath + if g.encryption != nil { + decPath := filepath.Join(tmpDir, "floop-backup-decrypted.json.gz") + if err := DecryptFile(g.encryption.IdentityFile, downloadPath, decPath); err != nil { + return nil, fmt.Errorf("decrypting backup: %w", err) + } + restorePath = decPath + } + + // Restore with merge semantics + restoreResult, err := backup.Restore(ctx, graphStore, restorePath, backup.RestoreMerge) + if err != nil { + return nil, fmt.Errorf("restoring backup: %w", err) + } + result.NodeCount = restoreResult.NodesRestored + result.EdgeCount = restoreResult.EdgesRestored + + // Download and merge corrections + if correctionsPath != "" { + corrKey := fmt.Sprintf("machines/%s/graph/corrections.jsonl", fromMachineID) + corrDownloadPath := filepath.Join(tmpDir, "corrections.jsonl") + + if err := g.downloadFile(ctx, corrKey, corrDownloadPath); err == nil { + corrRestorePath := corrDownloadPath + if g.encryption != nil { + decCorrPath := filepath.Join(tmpDir, "corrections-decrypted.jsonl") + if decErr := DecryptFile(g.encryption.IdentityFile, corrDownloadPath, decCorrPath); decErr != nil { + return nil, fmt.Errorf("decrypting corrections: %w", decErr) + } + corrRestorePath = decCorrPath + } + size, mergeErr := mergeCorrections(corrRestorePath, correctionsPath) + if mergeErr != nil { + return nil, fmt.Errorf("merging corrections: %w", mergeErr) + } + result.CorrectionsSize = size + } + // If corrections don't exist remotely, that's fine — skip silently + } + + return result, nil +} + +// uploadFile opens a file and uploads it via S3. +func (g *GraphSyncer) uploadFile(ctx context.Context, key, path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return fmt.Errorf("stat %s: %w", path, err) + } + + return g.s3.Upload(ctx, key, f, info.Size()) +} + +// downloadFile downloads from S3 and writes to a local file. +func (g *GraphSyncer) downloadFile(ctx context.Context, key, path string) error { + rc, err := g.s3.Download(ctx, key) + if err != nil { + return err + } + defer rc.Close() + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("creating directory: %w", err) + } + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating file %s: %w", path, err) + } + defer f.Close() + + if _, err := io.Copy(f, rc); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + return nil +} + +// mergeCorrections appends lines from remotePath that don't exist in localPath. +// Uses content-based deduplication to handle cross-machine pulls safely. +func mergeCorrections(remotePath, localPath string) (int64, error) { + // Build set of existing local lines for dedup + existing := make(map[string]struct{}) + if f, err := os.Open(localPath); err == nil { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + existing[scanner.Text()] = struct{}{} + } + f.Close() + } + + // Read remote lines, collect those not already in local + remoteFile, err := os.Open(remotePath) + if err != nil { + return 0, fmt.Errorf("opening remote corrections: %w", err) + } + defer remoteFile.Close() + + var newLines []string + scanner := bufio.NewScanner(remoteFile) + for scanner.Scan() { + line := scanner.Text() + if _, ok := existing[line]; !ok { + newLines = append(newLines, line) + } + } + if err := scanner.Err(); err != nil { + return 0, fmt.Errorf("scanning remote corrections: %w", err) + } + + if len(newLines) == 0 { + info, _ := os.Stat(localPath) + if info != nil { + return info.Size(), nil + } + return 0, nil + } + + // Append new lines to local file + dir := filepath.Dir(localPath) + if err := os.MkdirAll(dir, 0700); err != nil { + return 0, fmt.Errorf("creating corrections directory: %w", err) + } + + f, err := os.OpenFile(localPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return 0, fmt.Errorf("opening local corrections for append: %w", err) + } + defer f.Close() + + for _, line := range newLines { + if _, err := f.WriteString(line + "\n"); err != nil { + return 0, fmt.Errorf("appending correction: %w", err) + } + } + + info, _ := os.Stat(localPath) + if info != nil { + return info.Size(), nil + } + return 0, nil +} diff --git a/internal/vault/graph_sync_test.go b/internal/vault/graph_sync_test.go new file mode 100644 index 00000000..8be1432a --- /dev/null +++ b/internal/vault/graph_sync_test.go @@ -0,0 +1,288 @@ +package vault + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/nvandessel/floop/internal/store" +) + +// mockS3 is a simple in-memory S3 mock for testing. +type mockS3 struct { + mu sync.Mutex + objects map[string][]byte +} + +func newMockS3() *mockS3 { + return &mockS3{objects: make(map[string][]byte)} +} + +func (m *mockS3) Upload(_ context.Context, key string, reader io.Reader, _ int64) error { + m.mu.Lock() + defer m.mu.Unlock() + data, err := io.ReadAll(reader) + if err != nil { + return err + } + m.objects[key] = data + return nil +} + +func (m *mockS3) Download(_ context.Context, key string) (io.ReadCloser, error) { + m.mu.Lock() + defer m.mu.Unlock() + data, ok := m.objects[key] + if !ok { + return nil, fmt.Errorf("key %q not found", key) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (m *mockS3) Exists(_ context.Context, key string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.objects[key] + return ok, nil +} + +func (m *mockS3) PutJSON(_ context.Context, key string, v interface{}) error { + m.mu.Lock() + defer m.mu.Unlock() + data, err := json.Marshal(v) + if err != nil { + return err + } + m.objects[key] = data + return nil +} + +func (m *mockS3) GetJSON(_ context.Context, key string, v interface{}) error { + m.mu.Lock() + defer m.mu.Unlock() + data, ok := m.objects[key] + if !ok { + return fmt.Errorf("key %q not found", key) + } + return json.Unmarshal(data, v) +} + +func TestGraphSyncer_Push_UploadsBackup(t *testing.T) { + s3 := newMockS3() + syncer := NewGraphSyncer(s3, "workstation", nil) + + ctx := context.Background() + graphStore := newTestGraphStore(t) + + result, err := syncer.Push(ctx, graphStore, "", "1.0.0") + if err != nil { + t.Fatalf("Push: %v", err) + } + + // Verify backup was uploaded + backupKey := "machines/workstation/graph/floop-backup.json.gz" + exists, _ := s3.Exists(ctx, backupKey) + if !exists { + t.Error("backup was not uploaded to expected key") + } + + _ = result +} + +func TestGraphSyncer_Push_UploadsCorrections(t *testing.T) { + s3 := newMockS3() + syncer := NewGraphSyncer(s3, "workstation", nil) + + ctx := context.Background() + graphStore := newTestGraphStore(t) + + // Create a corrections file + corrDir := t.TempDir() + corrPath := filepath.Join(corrDir, "corrections.jsonl") + os.WriteFile(corrPath, []byte("line1\nline2\nline3\n"), 0600) + + result, err := syncer.Push(ctx, graphStore, corrPath, "1.0.0") + if err != nil { + t.Fatalf("Push: %v", err) + } + + corrKey := "machines/workstation/graph/corrections.jsonl" + exists, _ := s3.Exists(ctx, corrKey) + if !exists { + t.Error("corrections were not uploaded") + } + + if result.CorrectionsSize == 0 { + t.Error("CorrectionsSize should be > 0") + } +} + +func TestGraphSyncer_Pull_RestoresBackup(t *testing.T) { + s3 := newMockS3() + ctx := context.Background() + + // Push from "machine A" + graphStoreA := newTestGraphStore(t) + pushSyncer := NewGraphSyncer(s3, "machineA", nil) + _, err := pushSyncer.Push(ctx, graphStoreA, "", "1.0.0") + if err != nil { + t.Fatalf("Push: %v", err) + } + + // Pull to "machine B" + graphStoreB := newTestGraphStore(t) + pullSyncer := NewGraphSyncer(s3, "machineB", nil) + _, err = pullSyncer.Pull(ctx, graphStoreB, "machineA", "") + if err != nil { + t.Fatalf("Pull: %v", err) + } +} + +func TestMergeCorrections(t *testing.T) { + dir := t.TempDir() + + // Local has 5 lines + localPath := filepath.Join(dir, "local-corrections.jsonl") + var localLines []string + for i := 1; i <= 5; i++ { + localLines = append(localLines, fmt.Sprintf(`{"line":%d}`, i)) + } + os.WriteFile(localPath, []byte(strings.Join(localLines, "\n")+"\n"), 0600) + + // Remote has 8 lines (same first 5 + 3 new) + remotePath := filepath.Join(dir, "remote-corrections.jsonl") + var remoteLines []string + for i := 1; i <= 8; i++ { + remoteLines = append(remoteLines, fmt.Sprintf(`{"line":%d}`, i)) + } + os.WriteFile(remotePath, []byte(strings.Join(remoteLines, "\n")+"\n"), 0600) + + _, err := mergeCorrections(remotePath, localPath) + if err != nil { + t.Fatalf("mergeCorrections: %v", err) + } + + // Read local and count lines + data, _ := os.ReadFile(localPath) + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if len(lines) != 8 { + t.Errorf("local line count = %d, want 8", len(lines)) + } +} + +func TestMergeCorrections_CrossMachine(t *testing.T) { + dir := t.TempDir() + + // Local has corrections unique to this machine + localPath := filepath.Join(dir, "local.jsonl") + localContent := `{"src":"laptop","id":1} +{"src":"laptop","id":2} +` + os.WriteFile(localPath, []byte(localContent), 0600) + + // Remote has different corrections from another machine + remotePath := filepath.Join(dir, "remote.jsonl") + remoteContent := `{"src":"desktop","id":1} +{"src":"desktop","id":2} +{"src":"desktop","id":3} +` + os.WriteFile(remotePath, []byte(remoteContent), 0600) + + _, err := mergeCorrections(remotePath, localPath) + if err != nil { + t.Fatalf("mergeCorrections: %v", err) + } + + data, _ := os.ReadFile(localPath) + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + // Should have all 5 lines: 2 local + 3 remote (no overlap) + if len(lines) != 5 { + t.Errorf("line count = %d, want 5; content:\n%s", len(lines), string(data)) + } +} + +func TestMergeCorrections_NoNewLines(t *testing.T) { + dir := t.TempDir() + + content := "line1\nline2\nline3\n" + localPath := filepath.Join(dir, "local.jsonl") + remotePath := filepath.Join(dir, "remote.jsonl") + os.WriteFile(localPath, []byte(content), 0600) + os.WriteFile(remotePath, []byte(content), 0600) + + beforeData, _ := os.ReadFile(localPath) + + _, err := mergeCorrections(remotePath, localPath) + if err != nil { + t.Fatalf("mergeCorrections: %v", err) + } + + afterData, _ := os.ReadFile(localPath) + if string(afterData) != string(beforeData) { + t.Errorf("file was modified when no new lines should be added") + } +} + +// newTestGraphStore creates a minimal graph store for testing. +func newTestGraphStore(t *testing.T) *testGraphStore { + t.Helper() + return &testGraphStore{} +} + +// testGraphStore is a minimal in-memory graph store for graph sync tests. +type testGraphStore struct { + nodes []store.Node + edges []store.Edge +} + +func (s *testGraphStore) AddNode(_ context.Context, node store.Node) (string, error) { + s.nodes = append(s.nodes, node) + return node.ID, nil +} + +func (s *testGraphStore) UpdateNode(_ context.Context, _ store.Node) error { return nil } + +func (s *testGraphStore) GetNode(_ context.Context, id string) (*store.Node, error) { + for _, n := range s.nodes { + if n.ID == id { + return &n, nil + } + } + return nil, nil +} + +func (s *testGraphStore) DeleteNode(_ context.Context, _ string) error { return nil } + +func (s *testGraphStore) QueryNodes(_ context.Context, _ map[string]interface{}) ([]store.Node, error) { + return s.nodes, nil +} + +func (s *testGraphStore) AddEdge(_ context.Context, edge store.Edge) error { + s.edges = append(s.edges, edge) + return nil +} + +func (s *testGraphStore) RemoveEdge(_ context.Context, _, _ string, _ store.EdgeKind) error { + return nil +} + +func (s *testGraphStore) GetEdges(_ context.Context, _ string, _ store.Direction, _ store.EdgeKind) ([]store.Edge, error) { + return nil, nil +} + +func (s *testGraphStore) Traverse(_ context.Context, _ string, _ []store.EdgeKind, _ store.Direction, _ int) ([]store.Node, error) { + return nil, nil +} + +func (s *testGraphStore) Sync(_ context.Context) error { return nil } +func (s *testGraphStore) Close() error { return nil } + +// Verify testGraphStore implements GraphStore. +var _ store.GraphStore = (*testGraphStore)(nil) diff --git a/internal/vault/s3client.go b/internal/vault/s3client.go new file mode 100644 index 00000000..83ae7dde --- /dev/null +++ b/internal/vault/s3client.go @@ -0,0 +1,152 @@ +package vault + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/url" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// S3Operations defines the interface for S3 file operations. +// Used for testability of components that depend on S3. +type S3Operations interface { + Upload(ctx context.Context, key string, reader io.Reader, size int64) error + Download(ctx context.Context, key string) (io.ReadCloser, error) + Exists(ctx context.Context, key string) (bool, error) + PutJSON(ctx context.Context, key string, v interface{}) error + GetJSON(ctx context.Context, key string, v interface{}) error +} + +// S3Client wraps the MinIO Go SDK for file upload/download to S3-compatible storage. +type S3Client struct { + client *minio.Client + bucket string + prefix string +} + +// NewS3Client creates an S3Client from vault remote config. +func NewS3Client(cfg VaultRemoteConfig) (*S3Client, error) { + bucket, prefix := parseS3URI(cfg.URI) + if bucket == "" { + return nil, fmt.Errorf("invalid S3 URI: must contain a bucket name") + } + + endpoint := cfg.Endpoint + if endpoint == "" { + return nil, fmt.Errorf("endpoint is required for S3Client") + } + + // Parse endpoint to extract host (without scheme) + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("parsing endpoint URL: %w", err) + } + host := u.Host + if host == "" { + host = endpoint // fallback if no scheme + } + useSSL := !cfg.AllowHTTP && u.Scheme != "http" + + client, err := minio.New(host, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + return nil, fmt.Errorf("creating MinIO client: %w", err) + } + + return &S3Client{ + client: client, + bucket: bucket, + prefix: prefix, + }, nil +} + +// fullKey returns the full S3 object key by joining prefix and key. +func (c *S3Client) fullKey(key string) string { + if c.prefix == "" { + return key + } + return c.prefix + "/" + key +} + +// Upload uploads data to the given key. +func (c *S3Client) Upload(ctx context.Context, key string, reader io.Reader, size int64) error { + _, err := c.client.PutObject(ctx, c.bucket, c.fullKey(key), reader, size, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("uploading %s: %w", key, err) + } + return nil +} + +// Download downloads the object at the given key. +func (c *S3Client) Download(ctx context.Context, key string) (io.ReadCloser, error) { + obj, err := c.client.GetObject(ctx, c.bucket, c.fullKey(key), minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", key, err) + } + // Verify the object exists by stat'ing it + if _, err := obj.Stat(); err != nil { + obj.Close() + return nil, fmt.Errorf("downloading %s: %w", key, err) + } + return obj, nil +} + +// Exists checks if an object exists at the given key. +func (c *S3Client) Exists(ctx context.Context, key string) (bool, error) { + _, err := c.client.StatObject(ctx, c.bucket, c.fullKey(key), minio.StatObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.Code == "NoSuchKey" || errResp.StatusCode == 404 { + return false, nil + } + return false, fmt.Errorf("checking existence of %s: %w", key, err) + } + return true, nil +} + +// PutJSON marshals v to JSON and uploads it. +func (c *S3Client) PutJSON(ctx context.Context, key string, v interface{}) error { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Errorf("marshaling JSON for %s: %w", key, err) + } + return c.Upload(ctx, key, bytes.NewReader(data), int64(len(data))) +} + +// GetJSON downloads JSON from the given key and unmarshals it into v. +func (c *S3Client) GetJSON(ctx context.Context, key string, v interface{}) error { + rc, err := c.Download(ctx, key) + if err != nil { + return err + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return fmt.Errorf("reading %s: %w", key, err) + } + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("parsing JSON from %s: %w", key, err) + } + return nil +} + +// Bucket returns the bucket name. +func (c *S3Client) Bucket() string { + return c.bucket +} + +// Prefix returns the key prefix. +func (c *S3Client) Prefix() string { + return c.prefix +} + +// Verify S3Client satisfies S3Operations at compile time. +var _ S3Operations = (*S3Client)(nil) diff --git a/internal/vault/s3client_test.go b/internal/vault/s3client_test.go new file mode 100644 index 00000000..1eb07c6a --- /dev/null +++ b/internal/vault/s3client_test.go @@ -0,0 +1,84 @@ +package vault + +import "testing" + +func TestNewS3Client_URIParsing(t *testing.T) { + tests := []struct { + name string + uri string + wantBucket string + wantPrefix string + wantErr bool + }{ + {"bucket only", "s3://mybucket", "mybucket", "", false}, + {"bucket with prefix", "s3://mybucket/prefix", "mybucket", "prefix", false}, + {"bucket with deep prefix", "s3://mybucket/deep/prefix", "mybucket", "deep/prefix", false}, + {"empty bucket", "s3://", "", "", true}, + {"invalid scheme", "http://bucket", "", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := VaultRemoteConfig{ + URI: tt.uri, + Endpoint: "http://localhost:9000", + Region: "us-east-1", + AccessKeyID: "minioadmin", + SecretAccessKey: "minioadmin", + AllowHTTP: true, + } + + client, err := NewS3Client(cfg) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if client.Bucket() != tt.wantBucket { + t.Errorf("bucket = %q, want %q", client.Bucket(), tt.wantBucket) + } + if client.Prefix() != tt.wantPrefix { + t.Errorf("prefix = %q, want %q", client.Prefix(), tt.wantPrefix) + } + }) + } +} + +func TestNewS3Client_MissingEndpoint(t *testing.T) { + cfg := VaultRemoteConfig{ + URI: "s3://bucket", + Region: "us-east-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + } + _, err := NewS3Client(cfg) + if err == nil { + t.Fatal("expected error for missing endpoint") + } +} + +func TestS3Client_FullKey(t *testing.T) { + tests := []struct { + prefix string + key string + want string + }{ + {"", "file.txt", "file.txt"}, + {"brain", "file.txt", "brain/file.txt"}, + {"deep/prefix", "file.txt", "deep/prefix/file.txt"}, + } + + for _, tt := range tests { + t.Run(tt.prefix+"/"+tt.key, func(t *testing.T) { + c := &S3Client{prefix: tt.prefix} + if got := c.fullKey(tt.key); got != tt.want { + t.Errorf("fullKey(%q) = %q, want %q", tt.key, got, tt.want) + } + }) + } +} diff --git a/internal/vault/state.go b/internal/vault/state.go new file mode 100644 index 00000000..9d6abcc0 --- /dev/null +++ b/internal/vault/state.go @@ -0,0 +1,77 @@ +package vault + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" +) + +// SyncState tracks the last sync state for vault operations. +type SyncState struct { + MachineID string `json:"machine_id"` + LastPush time.Time `json:"last_push,omitempty"` + LastPull time.Time `json:"last_pull,omitempty"` + LocalVectorRows int `json:"local_vector_rows"` + RemoteVectorRows int `json:"remote_vector_rows"` + PushCount int `json:"push_count"` + PullCount int `json:"pull_count"` + PendingPush bool `json:"pending_push,omitempty"` +} + +// LoadState reads sync state from path. Returns zero state if file is missing. +func LoadState(path string) (*SyncState, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &SyncState{}, nil + } + return nil, fmt.Errorf("reading sync state: %w", err) + } + var state SyncState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("parsing sync state: %w", err) + } + return &state, nil +} + +// SaveState writes sync state atomically (write to temp, then rename). +func SaveState(path string, state *SyncState) error { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("marshaling sync state: %w", err) + } + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("creating state directory: %w", err) + } + + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0600); err != nil { + return fmt.Errorf("writing temp state: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("renaming state file: %w", err) + } + return nil +} + +// Staleness returns the staleness category based on the last push time. +// Takes now as a parameter for deterministic testing. +func (s *SyncState) Staleness(now time.Time) string { + if s.LastPush.IsZero() { + return "very_stale" + } + age := now.Sub(s.LastPush) + switch { + case age > 7*24*time.Hour: + return "very_stale" + case age > 24*time.Hour: + return "stale" + default: + return "fresh" + } +} diff --git a/internal/vault/state_test.go b/internal/vault/state_test.go new file mode 100644 index 00000000..94982fdb --- /dev/null +++ b/internal/vault/state_test.go @@ -0,0 +1,110 @@ +package vault + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestSyncState_RoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "vault-state.json") + + want := &SyncState{ + MachineID: "workstation", + LastPush: time.Date(2026, 4, 14, 10, 30, 0, 0, time.UTC), + LastPull: time.Date(2026, 4, 14, 8, 0, 0, 0, time.UTC), + LocalVectorRows: 142, + RemoteVectorRows: 140, + PushCount: 42, + PullCount: 38, + } + + if err := SaveState(path, want); err != nil { + t.Fatalf("SaveState: %v", err) + } + + got, err := LoadState(path) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + + if got.MachineID != want.MachineID { + t.Errorf("MachineID = %q, want %q", got.MachineID, want.MachineID) + } + if !got.LastPush.Equal(want.LastPush) { + t.Errorf("LastPush = %v, want %v", got.LastPush, want.LastPush) + } + if !got.LastPull.Equal(want.LastPull) { + t.Errorf("LastPull = %v, want %v", got.LastPull, want.LastPull) + } + if got.LocalVectorRows != want.LocalVectorRows { + t.Errorf("LocalVectorRows = %d, want %d", got.LocalVectorRows, want.LocalVectorRows) + } + if got.PushCount != want.PushCount { + t.Errorf("PushCount = %d, want %d", got.PushCount, want.PushCount) + } +} + +func TestSyncState_MissingFile(t *testing.T) { + got, err := LoadState(filepath.Join(t.TempDir(), "nonexistent.json")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.MachineID != "" { + t.Errorf("expected zero state, got MachineID=%q", got.MachineID) + } + if got.PushCount != 0 { + t.Errorf("expected zero state, got PushCount=%d", got.PushCount) + } +} + +func TestSyncState_AtomicWrite(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "vault-state.json") + + state := &SyncState{MachineID: "test"} + if err := SaveState(path, state); err != nil { + t.Fatalf("SaveState: %v", err) + } + + // Verify temp file is cleaned up + tmpPath := path + ".tmp" + if _, err := os.Stat(tmpPath); !os.IsNotExist(err) { + t.Errorf("temp file should not exist after successful save") + } + + // Verify final file exists + if _, err := os.Stat(path); err != nil { + t.Errorf("state file should exist: %v", err) + } +} + +func TestSyncState_Staleness(t *testing.T) { + now := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) + + tests := []struct { + name string + lastPush time.Time + want string + }{ + {"never pushed", time.Time{}, "very_stale"}, + {"30 min ago", now.Add(-30 * time.Minute), "fresh"}, + {"2 hours ago", now.Add(-2 * time.Hour), "fresh"}, + {"23 hours ago", now.Add(-23 * time.Hour), "fresh"}, + {"25 hours ago", now.Add(-25 * time.Hour), "stale"}, + {"3 days ago", now.Add(-3 * 24 * time.Hour), "stale"}, + {"8 days ago", now.Add(-8 * 24 * time.Hour), "very_stale"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &SyncState{LastPush: tt.lastPush} + got := s.Staleness(now) + if got != tt.want { + t.Errorf("Staleness() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/vault/vault.go b/internal/vault/vault.go new file mode 100644 index 00000000..26b45313 --- /dev/null +++ b/internal/vault/vault.go @@ -0,0 +1,534 @@ +package vault + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/lancedb/lancedb-go/pkg/contracts" + + "github.com/nvandessel/floop/internal/store" +) + +const ( + sentinelKey = "_floop_vault_initialized" + schemaVersionKey = "_schema_version" + syncStateKey = "_sync_state.json" + + // CurrentSchemaVersion is the vault schema version. + CurrentSchemaVersion = 1 + + // Scope constants for push/pull/sync operations. + ScopeGlobal = "global" + ScopeLocal = "local" + ScopeBoth = "both" +) + +// PushResult contains the results of a push operation. +type PushResult struct { + Vectors VectorSyncResult + Graph GraphSyncResult + Duration time.Duration +} + +// PullResult contains the results of a pull operation. +type PullResult struct { + Vectors VectorSyncResult + Graph GraphSyncResult + Duration time.Duration +} + +// SyncResult contains the results of a bidirectional sync. +type SyncResult struct { + Pulled PullResult + Pushed PushResult +} + +// StatusResult contains vault status information. +type StatusResult struct { + Configured bool `json:"configured"` + URI string `json:"uri"` + MachineID string `json:"machine_id"` + LastPush time.Time `json:"last_push,omitempty"` + LastPull time.Time `json:"last_pull,omitempty"` + LocalVectorRows int `json:"local_vector_rows"` + RemoteVectorRows int `json:"remote_vector_rows"` + LocalNodeCount int `json:"local_node_count"` + Status string `json:"status"` + Staleness string `json:"staleness"` +} + +// PushOptions controls push behavior. +type PushOptions struct { + Force bool + DryRun bool + Scope string // ScopeGlobal, ScopeLocal, or ScopeBoth +} + +// PullOptions controls pull behavior. +type PullOptions struct { + Force bool + DryRun bool + FromMachine string + Scope string + Root string +} + +// SyncOptions controls sync behavior. +type SyncOptions struct { + DryRun bool + Scope string +} + +// VerifyOptions controls verify behavior. +type VerifyOptions struct { + RemoteOnly bool + LocalOnly bool +} + +// VerifyResult contains verification results. +type VerifyResult struct { + OK bool `json:"ok"` + Issues []string `json:"issues,omitempty"` + LocalVectorRows int `json:"local_vector_rows"` + RemoteVectorRows int `json:"remote_vector_rows"` +} + +// VaultService orchestrates all vault sync operations. +type VaultService struct { + cfg *VaultConfig + s3 *S3Client + vectorDir string + floopVersion string + statePath string + dims int +} + +// NewVaultService creates a VaultService. +func NewVaultService(cfg *VaultConfig, vectorDir string, floopVersion string, dims int) (*VaultService, error) { + if !cfg.Configured() { + return nil, fmt.Errorf("vault not configured — run 'floop vault init'") + } + + s3, err := NewS3Client(cfg.Remote) + if err != nil { + return nil, fmt.Errorf("creating S3 client: %w", err) + } + + homeDir, _ := os.UserHomeDir() + statePath := filepath.Join(homeDir, ".floop", "vault-state.json") + + return &VaultService{ + cfg: cfg, + s3: s3, + vectorDir: vectorDir, + floopVersion: floopVersion, + statePath: statePath, + dims: dims, + }, nil +} + +// Init validates config, tests connectivity, and writes sentinel. +func (v *VaultService) Init(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, v.cfg.SyncTimeout()) + defer cancel() + + // Test connectivity by writing sentinel + if err := v.s3.PutJSON(ctx, sentinelKey, map[string]interface{}{ + "initialized_at": time.Now().UTC(), + "machine_id": v.cfg.ResolveMachineID(), + }); err != nil { + return fmt.Errorf("cannot connect to remote: %w", err) + } + + // Write schema version + if err := v.s3.PutJSON(ctx, schemaVersionKey, map[string]interface{}{ + "version": CurrentSchemaVersion, + }); err != nil { + return fmt.Errorf("writing schema version: %w", err) + } + + return nil +} + +// Push pushes local state to remote. +func (v *VaultService) Push(ctx context.Context, graphStore store.GraphStore, root string, opts PushOptions) (*PushResult, error) { + start := time.Now() + ctx, cancel := context.WithTimeout(ctx, v.cfg.SyncTimeout()) + defer cancel() + + machineID := v.cfg.ResolveMachineID() + result := &PushResult{} + + scope := normalizeScope(opts.Scope) + + if opts.DryRun { + return v.dryRunPush(ctx, graphStore, root, scope) + } + + if scope == ScopeGlobal || scope == ScopeBoth { + if err := v.pushScope(ctx, graphStore, machineID, v.vectorDir, v.globalCorrectionsPath(), result); err != nil { + return nil, err + } + } + + if (scope == ScopeLocal || scope == ScopeBoth) && root != "" { + localVectorDir := filepath.Join(root, ".floop", "vectors") + localCorrectionsPath := filepath.Join(root, ".floop", "corrections.jsonl") + if err := v.pushScope(ctx, graphStore, machineID, localVectorDir, localCorrectionsPath, result); err != nil { + return nil, err + } + } + + // Update state + state, err := LoadState(v.statePath) + if err != nil || state == nil { + state = &SyncState{} + } + state.MachineID = machineID + state.LastPush = time.Now().UTC() + state.LocalVectorRows = result.Vectors.RowsPushed + state.PushCount++ + state.PendingPush = false + if err := SaveState(v.statePath, state); err != nil { + fmt.Fprintf(os.Stderr, "warning: saving vault state: %v\n", err) + } + + // Write remote sync state (best-effort, don't fail the push) + if err := v.s3.PutJSON(ctx, fmt.Sprintf("machines/%s/%s", machineID, syncStateKey), state); err != nil { + fmt.Fprintf(os.Stderr, "warning: writing remote sync state: %v\n", err) + } + + result.Duration = time.Since(start) + return result, nil +} + +// Pull pulls remote state to local. +func (v *VaultService) Pull(ctx context.Context, graphStore store.GraphStore, opts PullOptions) (*PullResult, error) { + start := time.Now() + ctx, cancel := context.WithTimeout(ctx, v.cfg.SyncTimeout()) + defer cancel() + + machineID := v.cfg.ResolveMachineID() + fromMachine := opts.FromMachine + if fromMachine == "" { + fromMachine = machineID + } + + result := &PullResult{} + + if opts.DryRun { + return v.dryRunPull(ctx, graphStore, fromMachine, opts.Scope, opts.Root) + } + + scope := normalizeScope(opts.Scope) + + if scope == ScopeGlobal || scope == ScopeBoth { + if err := v.pullScope(ctx, graphStore, fromMachine, v.vectorDir, v.globalCorrectionsPath(), result); err != nil { + return nil, err + } + } + + if (scope == ScopeLocal || scope == ScopeBoth) && opts.Root != "" { + localVectorDir := filepath.Join(opts.Root, ".floop", "vectors") + localCorrectionsPath := filepath.Join(opts.Root, ".floop", "corrections.jsonl") + if err := v.pullScope(ctx, graphStore, fromMachine, localVectorDir, localCorrectionsPath, result); err != nil { + return nil, err + } + } + + // Update state + state, err := LoadState(v.statePath) + if err != nil || state == nil { + state = &SyncState{} + } + state.MachineID = machineID + state.LastPull = time.Now().UTC() + state.PullCount++ + if err := SaveState(v.statePath, state); err != nil { + fmt.Fprintf(os.Stderr, "warning: saving vault state: %v\n", err) + } + + result.Duration = time.Since(start) + return result, nil +} + +// Sync performs bidirectional sync (pull first, then push). +func (v *VaultService) Sync(ctx context.Context, graphStore store.GraphStore, root string, opts SyncOptions) (*SyncResult, error) { + result := &SyncResult{} + + pullResult, err := v.Pull(ctx, graphStore, PullOptions{ + DryRun: opts.DryRun, + Scope: opts.Scope, + Root: root, + }) + if err != nil { + return nil, fmt.Errorf("pull phase: %w", err) + } + result.Pulled = *pullResult + + pushResult, err := v.Push(ctx, graphStore, root, PushOptions{ + DryRun: opts.DryRun, + Scope: opts.Scope, + }) + if err != nil { + return nil, fmt.Errorf("push phase: %w", err) + } + result.Pushed = *pushResult + + return result, nil +} + +// Status returns the current vault status. +func (v *VaultService) Status(ctx context.Context, graphStore store.GraphStore) (*StatusResult, error) { + machineID := v.cfg.ResolveMachineID() + state, err := LoadState(v.statePath) + if err != nil || state == nil { + state = &SyncState{} + } + + sr := &StatusResult{ + Configured: true, + URI: v.cfg.Remote.URI, + MachineID: machineID, + LastPush: state.LastPush, + LastPull: state.LastPull, + Staleness: state.Staleness(time.Now()), + } + + // Count local nodes + nodes, err := graphStore.QueryNodes(ctx, map[string]interface{}{}) + if err == nil { + sr.LocalNodeCount = len(nodes) + } + + // Count local vector rows + remoteURI := v.remoteVectorURI(machineID) + opts := v.connectionOptions() + syncer := NewVectorSyncer(v.vectorDir, remoteURI, opts, v.dims) + + localCount, err := syncer.LocalRowCount(ctx) + if err == nil { + sr.LocalVectorRows = localCount + } + + remoteCount, err := syncer.RemoteRowCount(ctx) + if err == nil { + sr.RemoteVectorRows = remoteCount + } + + // Determine status + switch { + case sr.LocalVectorRows > sr.RemoteVectorRows: + sr.Status = "local_ahead" + case sr.RemoteVectorRows > sr.LocalVectorRows: + sr.Status = "remote_ahead" + default: + sr.Status = "in_sync" + } + + return sr, nil +} + +// Verify checks the integrity of local and/or remote vault data. +func (v *VaultService) Verify(ctx context.Context, graphStore store.GraphStore, opts VerifyOptions) (*VerifyResult, error) { + ctx, cancel := context.WithTimeout(ctx, v.cfg.SyncTimeout()) + defer cancel() + + machineID := v.cfg.ResolveMachineID() + result := &VerifyResult{ + OK: true, + LocalVectorRows: -1, + RemoteVectorRows: -1, + } + + remoteURI := v.remoteVectorURI(machineID) + connOpts := v.connectionOptions() + syncer := NewVectorSyncer(v.vectorDir, remoteURI, connOpts, v.dims) + + if !opts.RemoteOnly { + localCount, err := syncer.LocalRowCount(ctx) + if err != nil { + result.OK = false + result.Issues = append(result.Issues, fmt.Sprintf("local vectors unreadable: %v", err)) + } else { + result.LocalVectorRows = localCount + } + + nodes, err := graphStore.QueryNodes(ctx, map[string]interface{}{}) + if err != nil { + result.OK = false + result.Issues = append(result.Issues, fmt.Sprintf("local graph unreadable: %v", err)) + } else if len(nodes) == 0 { + result.Issues = append(result.Issues, "local graph is empty") + } + } + + if !opts.LocalOnly { + exists, err := v.s3.Exists(ctx, sentinelKey) + if err != nil { + result.OK = false + result.Issues = append(result.Issues, fmt.Sprintf("cannot reach remote: %v", err)) + } else if !exists { + result.OK = false + result.Issues = append(result.Issues, "remote vault not initialized (sentinel missing)") + } + + remoteCount, err := syncer.RemoteRowCount(ctx) + if err != nil { + result.OK = false + result.Issues = append(result.Issues, fmt.Sprintf("remote vectors unreadable: %v", err)) + } else { + result.RemoteVectorRows = remoteCount + } + } + + return result, nil +} + +// pushScope pushes vectors and graph for a single scope. +func (v *VaultService) pushScope(ctx context.Context, graphStore store.GraphStore, machineID, vectorDir, correctionsPath string, result *PushResult) error { + // Push vectors + remoteURI := v.remoteVectorURI(machineID) + opts := v.connectionOptions() + vectorSyncer := NewVectorSyncer(vectorDir, remoteURI, opts, v.dims) + + vecResult, err := vectorSyncer.Push(ctx) + if err != nil { + return fmt.Errorf("pushing vectors: %w", err) + } + result.Vectors.RowsPushed += vecResult.RowsPushed + result.Vectors.RowsSkipped += vecResult.RowsSkipped + + // Push graph + graphSyncer := NewGraphSyncer(v.s3, machineID, &v.cfg.Encryption) + graphResult, err := graphSyncer.Push(ctx, graphStore, correctionsPath, v.floopVersion) + if err != nil { + return fmt.Errorf("pushing graph: %w", err) + } + result.Graph.NodeCount += graphResult.NodeCount + result.Graph.EdgeCount += graphResult.EdgeCount + result.Graph.CorrectionsSize += graphResult.CorrectionsSize + + return nil +} + +// pullScope pulls vectors and graph for a single scope. +func (v *VaultService) pullScope(ctx context.Context, graphStore store.GraphStore, fromMachine, vectorDir, correctionsPath string, result *PullResult) error { + // Pull vectors + remoteURI := v.remoteVectorURI(fromMachine) + opts := v.connectionOptions() + vectorSyncer := NewVectorSyncer(vectorDir, remoteURI, opts, v.dims) + + vecResult, err := vectorSyncer.Pull(ctx) + if err != nil { + return fmt.Errorf("pulling vectors: %w", err) + } + result.Vectors.RowsPulled += vecResult.RowsPushed // syncRows reports pushed from source's perspective + result.Vectors.RowsSkipped += vecResult.RowsSkipped + + // Pull graph + graphSyncer := NewGraphSyncer(v.s3, v.cfg.ResolveMachineID(), &v.cfg.Encryption) + graphResult, err := graphSyncer.Pull(ctx, graphStore, fromMachine, correctionsPath) + if err != nil { + return fmt.Errorf("pulling graph: %w", err) + } + result.Graph.NodeCount += graphResult.NodeCount + result.Graph.EdgeCount += graphResult.EdgeCount + result.Graph.CorrectionsSize += graphResult.CorrectionsSize + + return nil +} + +// dryRunPush returns what would be pushed without pushing. +func (v *VaultService) dryRunPush(ctx context.Context, graphStore store.GraphStore, root, scope string) (*PushResult, error) { + result := &PushResult{} + machineID := v.cfg.ResolveMachineID() + remoteURI := v.remoteVectorURI(machineID) + connOpts := v.connectionOptions() + + if scope == ScopeGlobal || scope == ScopeBoth { + syncer := NewVectorSyncer(v.vectorDir, remoteURI, connOpts, v.dims) + localCount, err := syncer.LocalRowCount(ctx) + if err == nil { + result.Vectors.RowsPushed += localCount + } + } + + if (scope == ScopeLocal || scope == ScopeBoth) && root != "" { + localVectorDir := filepath.Join(root, ".floop", "vectors") + syncer := NewVectorSyncer(localVectorDir, remoteURI, connOpts, v.dims) + localCount, err := syncer.LocalRowCount(ctx) + if err == nil { + result.Vectors.RowsPushed += localCount + } + } + + // Count graph nodes + nodes, err := graphStore.QueryNodes(ctx, map[string]interface{}{}) + if err == nil { + result.Graph.NodeCount = len(nodes) + } + + return result, nil +} + +// dryRunPull returns what would be pulled without pulling. +func (v *VaultService) dryRunPull(ctx context.Context, graphStore store.GraphStore, fromMachine, scope, root string) (*PullResult, error) { + result := &PullResult{} + + remoteURI := v.remoteVectorURI(fromMachine) + connOpts := v.connectionOptions() + + if scope == ScopeGlobal || scope == ScopeBoth { + syncer := NewVectorSyncer(v.vectorDir, remoteURI, connOpts, v.dims) + remoteCount, err := syncer.RemoteRowCount(ctx) + if err == nil { + result.Vectors.RowsPulled += remoteCount + } + } + + if (scope == ScopeLocal || scope == ScopeBoth) && root != "" { + localVectorDir := filepath.Join(root, ".floop", "vectors") + syncer := NewVectorSyncer(localVectorDir, remoteURI, connOpts, v.dims) + remoteCount, err := syncer.RemoteRowCount(ctx) + if err == nil { + result.Vectors.RowsPulled += remoteCount + } + } + + return result, nil +} + +// remoteVectorURI builds the S3 URI for the vectors table. +func (v *VaultService) remoteVectorURI(machineID string) string { + return fmt.Sprintf("%s/machines/%s/vectors", v.cfg.Remote.URI, machineID) +} + +// connectionOptions builds lancedb-go connection options from config. +func (v *VaultService) connectionOptions() *contracts.ConnectionOptions { + storageOpts := v.cfg.Remote.StorageOptions() + return &contracts.ConnectionOptions{ + StorageOptions: storageOpts, + } +} + +// globalCorrectionsPath returns the path to the global corrections.jsonl. +func (v *VaultService) globalCorrectionsPath() string { + homeDir, _ := os.UserHomeDir() + return filepath.Join(homeDir, ".floop", "corrections.jsonl") +} + +// normalizeScope normalizes the scope string, defaulting to "global". +func normalizeScope(scope string) string { + scope = strings.ToLower(strings.TrimSpace(scope)) + switch scope { + case ScopeGlobal, ScopeLocal, ScopeBoth: + return scope + default: + return ScopeGlobal + } +} diff --git a/internal/vault/vault_integration_test.go b/internal/vault/vault_integration_test.go new file mode 100644 index 00000000..754baf8e --- /dev/null +++ b/internal/vault/vault_integration_test.go @@ -0,0 +1,203 @@ +//go:build integration + +package vault + +import ( + "bytes" + "context" + "fmt" + "io" + "path/filepath" + "testing" + "time" + + "github.com/minio/minio-go/v7" +) + +// Integration tests require MinIO running on localhost:9000. +// Run: docker compose -f docker-compose.vault-test.yml up -d +// Then: go test -tags integration -v ./internal/vault/... + +var testEndpoint = "http://localhost:9000" +var testBucket = "floop-vault-test" + +func testRemoteConfig(prefix string) VaultRemoteConfig { + return VaultRemoteConfig{ + URI: fmt.Sprintf("s3://%s/%s", testBucket, prefix), + Endpoint: testEndpoint, + Region: "us-east-1", + AccessKeyID: "minioadmin", + SecretAccessKey: "minioadmin", + PathStyle: true, + AllowHTTP: true, + } +} + +func TestIntegration_S3ClientRoundTrip(t *testing.T) { + prefix := fmt.Sprintf("test-s3-%d", time.Now().UnixNano()) + cfg := testRemoteConfig(prefix) + + // Ensure bucket exists + ensureTestBucket(t) + + client, err := NewS3Client(cfg) + if err != nil { + t.Fatalf("NewS3Client: %v", err) + } + + ctx := context.Background() + + // Upload + testData := []byte("hello vault integration test") + if err := client.Upload(ctx, "test-file.txt", newReader(testData), int64(len(testData))); err != nil { + t.Fatalf("Upload: %v", err) + } + + // Exists + exists, err := client.Exists(ctx, "test-file.txt") + if err != nil { + t.Fatalf("Exists: %v", err) + } + if !exists { + t.Fatal("file should exist after upload") + } + + // Download + rc, err := client.Download(ctx, "test-file.txt") + if err != nil { + t.Fatalf("Download: %v", err) + } + defer rc.Close() + + downloaded, err := readAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(downloaded) != string(testData) { + t.Errorf("downloaded = %q, want %q", string(downloaded), string(testData)) + } + + // PutJSON / GetJSON + type TestStruct struct { + Name string `json:"name"` + Count int `json:"count"` + } + want := TestStruct{Name: "test", Count: 42} + if err := client.PutJSON(ctx, "test.json", want); err != nil { + t.Fatalf("PutJSON: %v", err) + } + + var got TestStruct + if err := client.GetJSON(ctx, "test.json", &got); err != nil { + t.Fatalf("GetJSON: %v", err) + } + if got != want { + t.Errorf("GetJSON = %+v, want %+v", got, want) + } +} + +func TestIntegration_VaultInitConnectsToMinIO(t *testing.T) { + prefix := fmt.Sprintf("test-init-%d", time.Now().UnixNano()) + ensureTestBucket(t) + + cfg := &VaultConfig{ + Remote: testRemoteConfig(prefix), + MachineID: "test-machine", + Sync: VaultSyncConfig{Timeout: "30s"}, + } + + svc, err := NewVaultService(cfg, t.TempDir(), "test", 4) + if err != nil { + t.Fatalf("NewVaultService: %v", err) + } + + ctx := context.Background() + if err := svc.Init(ctx); err != nil { + t.Fatalf("Init: %v", err) + } + + // Verify sentinel was written + s3Client, _ := NewS3Client(cfg.Remote) + exists, _ := s3Client.Exists(ctx, "_floop_vault_initialized") + if !exists { + t.Error("sentinel should exist after Init") + } +} + +func TestIntegration_VaultPushAndPull(t *testing.T) { + prefix := fmt.Sprintf("test-pushpull-%d", time.Now().UnixNano()) + ensureTestBucket(t) + + cfg := &VaultConfig{ + Remote: testRemoteConfig(prefix), + MachineID: "machineA", + Sync: VaultSyncConfig{Timeout: "30s"}, + } + + // Create a graph store with some data + graphStore := newTestGraphStore(t) + + tmpDir := t.TempDir() + svc, err := NewVaultService(cfg, filepath.Join(tmpDir, "vectors"), "test", 4) + if err != nil { + t.Fatalf("NewVaultService: %v", err) + } + + ctx := context.Background() + + // Push + _, err = svc.Push(ctx, graphStore, tmpDir, PushOptions{Scope: "global"}) + if err != nil { + t.Fatalf("Push: %v", err) + } + + // Pull to a fresh store + graphStoreB := newTestGraphStore(t) + cfgB := *cfg + cfgB.MachineID = "machineB" + svcB, _ := NewVaultService(&cfgB, filepath.Join(t.TempDir(), "vectors"), "test", 4) + + _, err = svcB.Pull(ctx, graphStoreB, PullOptions{ + FromMachine: "machineA", + Scope: "global", + }) + if err != nil { + t.Fatalf("Pull: %v", err) + } +} + +// ensureTestBucket creates the test bucket via the MinIO client. +func ensureTestBucket(t *testing.T) { + t.Helper() + cfg := VaultRemoteConfig{ + URI: fmt.Sprintf("s3://%s", testBucket), + Endpoint: testEndpoint, + Region: "us-east-1", + AccessKeyID: "minioadmin", + SecretAccessKey: "minioadmin", + PathStyle: true, + AllowHTTP: true, + } + client, err := NewS3Client(cfg) + if err != nil { + t.Skipf("cannot connect to MinIO: %v (run docker compose -f docker-compose.vault-test.yml up -d)", err) + } + + ctx := context.Background() + err = client.client.MakeBucket(ctx, testBucket, minio.MakeBucketOptions{}) + if err != nil { + // Bucket may already exist, that's fine + exists, existErr := client.client.BucketExists(ctx, testBucket) + if existErr != nil || !exists { + t.Skipf("cannot create test bucket: %v", err) + } + } +} + +func newReader(data []byte) io.Reader { + return bytes.NewReader(data) +} + +func readAll(rc io.ReadCloser) ([]byte, error) { + return io.ReadAll(rc) +} diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go new file mode 100644 index 00000000..11216189 --- /dev/null +++ b/internal/vault/vault_test.go @@ -0,0 +1,57 @@ +package vault + +import ( + "testing" +) + +func TestNewVaultService_UnconfiguredReturnsError(t *testing.T) { + cfg := &VaultConfig{} + _, err := NewVaultService(cfg, "/tmp/vectors", "1.0.0", 384) + if err == nil { + t.Fatal("expected error for unconfigured vault") + } + if !contains(err.Error(), "not configured") { + t.Errorf("error should mention 'not configured': %v", err) + } +} + +func TestNormalizeScope(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"global", "global"}, + {"local", "local"}, + {"both", "both"}, + {"GLOBAL", "global"}, + {"", "global"}, + {"invalid", "global"}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := normalizeScope(tt.input) + if got != tt.want { + t.Errorf("normalizeScope(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestRemoteVectorURI(t *testing.T) { + cfg := &VaultConfig{ + Remote: VaultRemoteConfig{ + URI: "s3://floop-vault/brain", + Endpoint: "https://minio.example.com:9000", + Region: "us-east-1", + AccessKeyID: "key", + SecretAccessKey: "secret", + }, + } + + svc := &VaultService{cfg: cfg} + got := svc.remoteVectorURI("workstation") + want := "s3://floop-vault/brain/machines/workstation/vectors" + if got != want { + t.Errorf("remoteVectorURI = %q, want %q", got, want) + } +} diff --git a/internal/vault/vector_sync.go b/internal/vault/vector_sync.go new file mode 100644 index 00000000..af945108 --- /dev/null +++ b/internal/vault/vector_sync.go @@ -0,0 +1,294 @@ +//go:build cgo + +package vault + +import ( + "context" + "fmt" + "strings" + "sync" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/lancedb/lancedb-go/pkg/contracts" + "github.com/lancedb/lancedb-go/pkg/lancedb" + + "github.com/nvandessel/floop/internal/vectorindex" +) + +// extractVector converts vector values from LanceDB result maps to []float32. +func extractVector(v interface{}) []float32 { + switch val := v.(type) { + case []interface{}: + out := make([]float32, len(val)) + for i, x := range val { + switch f := x.(type) { + case float64: + out[i] = float32(f) + case float32: + out[i] = f + default: + return nil + } + } + return out + case []float32: + return val + case []float64: + out := make([]float32, len(val)) + for i, f := range val { + out[i] = float32(f) + } + return out + } + return nil +} + +const behaviorTableName = "behaviors" + +// VectorSyncer syncs LanceDB vector tables between local and remote storage. +type VectorSyncer struct { + localDir string + remoteURI string + remoteOpts *contracts.ConnectionOptions + dims int + mu sync.Mutex +} + +// VectorSyncResult contains the results of a vector sync operation. +type VectorSyncResult struct { + RowsPushed int + RowsPulled int + RowsSkipped int +} + +// NewVectorSyncer creates a VectorSyncer for the given local directory and remote URI. +func NewVectorSyncer(localDir, remoteURI string, opts *contracts.ConnectionOptions, dims int) *VectorSyncer { + return &VectorSyncer{ + localDir: localDir, + remoteURI: remoteURI, + remoteOpts: opts, + dims: dims, + } +} + +// Push reads all rows from the local behaviors table and upserts them into the remote. +func (v *VectorSyncer) Push(ctx context.Context) (*VectorSyncResult, error) { + v.mu.Lock() + defer v.mu.Unlock() + + localDB, err := lancedb.Connect(ctx, v.localDir, nil) + if err != nil { + return nil, fmt.Errorf("connecting to local LanceDB: %w", err) + } + defer localDB.Close() + + localTable, err := openTable(ctx, localDB, behaviorTableName) + if err != nil { + return nil, fmt.Errorf("opening local table: %w", err) + } + defer localTable.Close() + + remoteDB, err := lancedb.Connect(ctx, v.remoteURI, v.remoteOpts) + if err != nil { + return nil, fmt.Errorf("connecting to remote LanceDB: %w", err) + } + defer remoteDB.Close() + + remoteTable, err := openOrCreateTable(ctx, remoteDB, behaviorTableName, v.dims) + if err != nil { + return nil, fmt.Errorf("opening remote table: %w", err) + } + defer remoteTable.Close() + + return syncRows(ctx, localTable, remoteTable, v.dims) +} + +// Pull reads all rows from the remote behaviors table and upserts them into the local. +func (v *VectorSyncer) Pull(ctx context.Context) (*VectorSyncResult, error) { + v.mu.Lock() + defer v.mu.Unlock() + + localDB, err := lancedb.Connect(ctx, v.localDir, nil) + if err != nil { + return nil, fmt.Errorf("connecting to local LanceDB: %w", err) + } + defer localDB.Close() + + localTable, err := openOrCreateTable(ctx, localDB, behaviorTableName, v.dims) + if err != nil { + return nil, fmt.Errorf("opening local table: %w", err) + } + defer localTable.Close() + + remoteDB, err := lancedb.Connect(ctx, v.remoteURI, v.remoteOpts) + if err != nil { + return nil, fmt.Errorf("connecting to remote LanceDB: %w", err) + } + defer remoteDB.Close() + + remoteTable, err := openTable(ctx, remoteDB, behaviorTableName) + if err != nil { + return nil, fmt.Errorf("opening remote table: %w", err) + } + defer remoteTable.Close() + + return syncRows(ctx, remoteTable, localTable, v.dims) +} + +// LocalRowCount returns the number of rows in the local behaviors table. +func (v *VectorSyncer) LocalRowCount(ctx context.Context) (int, error) { + db, err := lancedb.Connect(ctx, v.localDir, nil) + if err != nil { + return 0, fmt.Errorf("connecting to local LanceDB: %w", err) + } + defer db.Close() + + table, err := openTable(ctx, db, behaviorTableName) + if err != nil { + return 0, nil // table doesn't exist → 0 rows + } + defer table.Close() + + count, err := table.Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting local rows: %w", err) + } + return int(count), nil +} + +// RemoteRowCount returns the number of rows in the remote behaviors table. +func (v *VectorSyncer) RemoteRowCount(ctx context.Context) (int, error) { + db, err := lancedb.Connect(ctx, v.remoteURI, v.remoteOpts) + if err != nil { + return 0, fmt.Errorf("connecting to remote LanceDB: %w", err) + } + defer db.Close() + + table, err := openTable(ctx, db, behaviorTableName) + if err != nil { + return 0, nil // table doesn't exist → 0 rows + } + defer table.Close() + + count, err := table.Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting remote rows: %w", err) + } + return int(count), nil +} + +// syncRows reads all rows from src and upserts them into dst. +func syncRows(ctx context.Context, src, dst contracts.ITable, dims int) (*VectorSyncResult, error) { + count, err := src.Count(ctx) + if err != nil { + return nil, fmt.Errorf("counting source rows: %w", err) + } + if count == 0 { + return &VectorSyncResult{}, nil + } + + // Full table scan via JSON Select — deterministic, returns every row. + // Query().Execute() uses Arrow IPC which has a multi-batch concatenation bug + // in ipcBytesToRecord (returns only the first batch when rows span fragments). + rows, err := src.Select(ctx, contracts.QueryConfig{}) + if err != nil { + return nil, fmt.Errorf("reading source rows: %w", err) + } + + arrowSchema, vectorType := vectorindex.BuildBehaviorSchema(dims) + result := &VectorSyncResult{} + + for _, row := range rows { + id, ok := row["id"].(string) + if !ok { + result.RowsSkipped++ + continue + } + + vec := extractVector(row["vector"]) + if vec == nil { + result.RowsSkipped++ + continue + } + + // Upsert: delete then add + escaped := strings.ReplaceAll(id, "'", "''") + if delErr := dst.Delete(ctx, fmt.Sprintf("id = '%s'", escaped)); delErr != nil { + return nil, fmt.Errorf("deleting stale row %s: %w", id, delErr) + } + + rec, err := buildRecord(arrowSchema, vectorType, id, vec) + if err != nil { + return nil, fmt.Errorf("building record for %s: %w", id, err) + } + + if err := dst.Add(ctx, rec, nil); err != nil { + rec.Release() + return nil, fmt.Errorf("adding row %s: %w", id, err) + } + rec.Release() + result.RowsPushed++ + } + + return result, nil +} + +// buildRecord creates a single-row Arrow record for a behavior embedding. +func buildRecord(schema *arrow.Schema, vectorType *arrow.FixedSizeListType, id string, vector []float32) (arrow.Record, error) { + pool := memory.NewGoAllocator() + + idBuilder := array.NewStringBuilder(pool) + defer idBuilder.Release() + idBuilder.Append(id) + idArray := idBuilder.NewArray() + defer idArray.Release() + + floatBuilder := array.NewFloat32Builder(pool) + defer floatBuilder.Release() + floatBuilder.AppendValues(vector, nil) + floatArray := floatBuilder.NewArray() + defer floatArray.Release() + + vectorData := array.NewData(vectorType, 1, []*memory.Buffer{nil}, []arrow.ArrayData{floatArray.Data()}, 0, 0) + defer vectorData.Release() + vectorArray := array.NewFixedSizeListData(vectorData) + defer vectorArray.Release() + + rec := array.NewRecord(schema, []arrow.Array{idArray, vectorArray}, 1) + return rec, nil +} + +// openTable opens an existing table. Returns error if table doesn't exist. +func openTable(ctx context.Context, db contracts.IConnection, name string) (contracts.ITable, error) { + names, err := db.TableNames(ctx) + if err != nil { + return nil, fmt.Errorf("listing tables: %w", err) + } + for _, n := range names { + if n == name { + return db.OpenTable(ctx, name) + } + } + return nil, fmt.Errorf("table %q not found", name) +} + +// openOrCreateTable opens a table, or creates it if it doesn't exist. +func openOrCreateTable(ctx context.Context, db contracts.IConnection, name string, dims int) (contracts.ITable, error) { + names, err := db.TableNames(ctx) + if err != nil { + return nil, fmt.Errorf("listing tables: %w", err) + } + for _, n := range names { + if n == name { + return db.OpenTable(ctx, name) + } + } + + lanceSchema, err := vectorindex.BuildLanceSchema(dims) + if err != nil { + return nil, fmt.Errorf("building schema: %w", err) + } + return db.CreateTable(ctx, name, lanceSchema) +} diff --git a/internal/vault/vector_sync_nocgo.go b/internal/vault/vector_sync_nocgo.go new file mode 100644 index 00000000..796f51cb --- /dev/null +++ b/internal/vault/vector_sync_nocgo.go @@ -0,0 +1,48 @@ +//go:build !cgo + +package vault + +import ( + "context" + "errors" + + "github.com/lancedb/lancedb-go/pkg/contracts" +) + +var errNoCGO = errors.New("vector sync requires CGO (lancedb)") + +// VectorSyncer syncs LanceDB vector tables between local and remote storage. +// This is the non-CGO stub. +type VectorSyncer struct{} + +// VectorSyncResult contains the results of a vector sync operation. +type VectorSyncResult struct { + RowsPushed int + RowsPulled int + RowsSkipped int +} + +// NewVectorSyncer returns a stub VectorSyncer when CGO is disabled. +func NewVectorSyncer(localDir, remoteURI string, opts *contracts.ConnectionOptions, dims int) *VectorSyncer { + return &VectorSyncer{} +} + +// Push is a stub that returns an error when CGO is disabled. +func (v *VectorSyncer) Push(ctx context.Context) (*VectorSyncResult, error) { + return nil, errNoCGO +} + +// Pull is a stub that returns an error when CGO is disabled. +func (v *VectorSyncer) Pull(ctx context.Context) (*VectorSyncResult, error) { + return nil, errNoCGO +} + +// LocalRowCount is a stub that returns an error when CGO is disabled. +func (v *VectorSyncer) LocalRowCount(ctx context.Context) (int, error) { + return 0, errNoCGO +} + +// RemoteRowCount is a stub that returns an error when CGO is disabled. +func (v *VectorSyncer) RemoteRowCount(ctx context.Context) (int, error) { + return 0, errNoCGO +} diff --git a/internal/vault/vector_sync_test.go b/internal/vault/vector_sync_test.go new file mode 100644 index 00000000..4946fb38 --- /dev/null +++ b/internal/vault/vector_sync_test.go @@ -0,0 +1,182 @@ +//go:build cgo + +package vault + +import ( + "context" + "testing" + + "github.com/lancedb/lancedb-go/pkg/lancedb" + + "github.com/nvandessel/floop/internal/vectorindex" +) + +const testDims = 4 + +func TestVectorSyncer_LocalToLocal(t *testing.T) { + ctx := context.Background() + srcDir := t.TempDir() + dstDir := t.TempDir() + + // Create source table with data + srcDB, err := lancedb.Connect(ctx, srcDir, nil) + if err != nil { + t.Fatalf("connect src: %v", err) + } + + lanceSchema, err := vectorindex.BuildLanceSchema(testDims) + if err != nil { + t.Fatalf("build schema: %v", err) + } + + srcTable, err := srcDB.CreateTable(ctx, behaviorTableName, lanceSchema) + if err != nil { + t.Fatalf("create table: %v", err) + } + + arrowSchema, vectorType := vectorindex.BuildBehaviorSchema(testDims) + + // Add test rows + for i, id := range []string{"b1", "b2", "b3"} { + vec := make([]float32, testDims) + vec[i] = 1.0 + rec, err := buildRecord(arrowSchema, vectorType, id, vec) + if err != nil { + t.Fatalf("build record: %v", err) + } + if err := srcTable.Add(ctx, rec, nil); err != nil { + rec.Release() + t.Fatalf("add row: %v", err) + } + rec.Release() + } + srcTable.Close() + srcDB.Close() + + // Sync src → dst + syncer := NewVectorSyncer(srcDir, dstDir, nil, testDims) + result, err := syncer.Push(ctx) + if err != nil { + t.Fatalf("Push: %v", err) + } + + if result.RowsPushed != 3 { + t.Errorf("RowsPushed = %d, want 3", result.RowsPushed) + } + + // Verify dst has 3 rows + dstCount, err := syncer.RemoteRowCount(ctx) + if err != nil { + t.Fatalf("RemoteRowCount: %v", err) + } + if dstCount != 3 { + t.Errorf("remote count = %d, want 3", dstCount) + } +} + +func TestVectorSyncer_Idempotent(t *testing.T) { + ctx := context.Background() + srcDir := t.TempDir() + dstDir := t.TempDir() + + // Create source table with data + srcDB, err := lancedb.Connect(ctx, srcDir, nil) + if err != nil { + t.Fatalf("connect: %v", err) + } + + lanceSchema, err := vectorindex.BuildLanceSchema(testDims) + if err != nil { + t.Fatalf("build schema: %v", err) + } + srcTable, err := srcDB.CreateTable(ctx, behaviorTableName, lanceSchema) + if err != nil { + t.Fatalf("create table: %v", err) + } + + arrowSchema, vectorType := vectorindex.BuildBehaviorSchema(testDims) + rec, _ := buildRecord(arrowSchema, vectorType, "b1", []float32{1, 0, 0, 0}) + srcTable.Add(ctx, rec, nil) + rec.Release() + srcTable.Close() + srcDB.Close() + + syncer := NewVectorSyncer(srcDir, dstDir, nil, testDims) + + // Push twice + syncer.Push(ctx) + result, err := syncer.Push(ctx) + if err != nil { + t.Fatalf("second Push: %v", err) + } + + // Count should still be 1 + count, _ := syncer.RemoteRowCount(ctx) + if count != 1 { + t.Errorf("count after double push = %d, want 1", count) + } + _ = result +} + +func TestVectorSyncer_EmptySource(t *testing.T) { + ctx := context.Background() + srcDir := t.TempDir() + dstDir := t.TempDir() + + // Create empty source table + srcDB, _ := lancedb.Connect(ctx, srcDir, nil) + lanceSchema, _ := vectorindex.BuildLanceSchema(testDims) + srcTable, _ := srcDB.CreateTable(ctx, behaviorTableName, lanceSchema) + srcTable.Close() + srcDB.Close() + + syncer := NewVectorSyncer(srcDir, dstDir, nil, testDims) + result, err := syncer.Push(ctx) + if err != nil { + t.Fatalf("Push empty: %v", err) + } + if result.RowsPushed != 0 { + t.Errorf("RowsPushed = %d, want 0", result.RowsPushed) + } +} + +func TestVectorSyncer_PullRoundTrip(t *testing.T) { + ctx := context.Background() + srcDir := t.TempDir() + remoteDir := t.TempDir() + dstDir := t.TempDir() + + // Create source and push to "remote" + srcDB, _ := lancedb.Connect(ctx, srcDir, nil) + lanceSchema, _ := vectorindex.BuildLanceSchema(testDims) + srcTable, _ := srcDB.CreateTable(ctx, behaviorTableName, lanceSchema) + + arrowSchema, vectorType := vectorindex.BuildBehaviorSchema(testDims) + for i, id := range []string{"b1", "b2"} { + vec := make([]float32, testDims) + vec[i] = 1.0 + rec, _ := buildRecord(arrowSchema, vectorType, id, vec) + srcTable.Add(ctx, rec, nil) + rec.Release() + } + srcTable.Close() + srcDB.Close() + + pushSyncer := NewVectorSyncer(srcDir, remoteDir, nil, testDims) + pushSyncer.Push(ctx) + + // Pull from "remote" to dst + pullSyncer := NewVectorSyncer(dstDir, remoteDir, nil, testDims) + result, err := pullSyncer.Pull(ctx) + if err != nil { + t.Fatalf("Pull: %v", err) + } + if result.RowsPushed != 2 { + t.Errorf("RowsPulled = %d, want 2", result.RowsPushed) + } + + localCount, _ := pullSyncer.LocalRowCount(ctx) + if localCount != 2 { + t.Errorf("local count after pull = %d, want 2", localCount) + } +} diff --git a/internal/vectorindex/lancedb.go b/internal/vectorindex/lancedb.go index 88fbba48..8bca6c2f 100644 --- a/internal/vectorindex/lancedb.go +++ b/internal/vectorindex/lancedb.go @@ -34,6 +34,26 @@ type LanceDBIndex struct { vectorType *arrow.FixedSizeListType } +// BuildLanceSchema builds the LanceDB schema for the behaviors table. +// Used for creating new tables. +func BuildLanceSchema(dims int) (contracts.ISchema, error) { + return lancedb.NewSchemaBuilder(). + AddStringField("id", false). + AddVectorField("vector", dims, contracts.VectorDataTypeFloat32, false). + Build() +} + +// BuildBehaviorSchema returns the canonical Arrow schema and vector type +// for the behaviors table. Used by both LanceDBIndex and vault sync. +func BuildBehaviorSchema(dims int) (*arrow.Schema, *arrow.FixedSizeListType) { + vectorType := arrow.FixedSizeListOf(int32(dims), arrow.PrimitiveTypes.Float32) + arrowSchema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.BinaryTypes.String}, + {Name: "vector", Type: vectorType}, + }, nil) + return arrowSchema, vectorType +} + // NewLanceDBIndex creates a LanceDBIndex backed by the given directory. // If a table already exists, it is opened; otherwise a new one is created. func NewLanceDBIndex(cfg LanceDBConfig) (*LanceDBIndex, error) { @@ -107,26 +127,19 @@ func NewLanceDBIndex(cfg LanceDBConfig) (*LanceDBIndex, error) { ) } } else { - schema, serr := lancedb.NewSchemaBuilder(). - AddStringField("id", false). - AddVectorField("vector", cfg.Dims, contracts.VectorDataTypeFloat32, false). - Build() + lanceSchema, serr := BuildLanceSchema(cfg.Dims) if serr != nil { db.Close() return nil, fmt.Errorf("build schema: %w", serr) } - table, err = db.CreateTable(ctx, lanceTableName, schema) + table, err = db.CreateTable(ctx, lanceTableName, lanceSchema) if err != nil { db.Close() return nil, fmt.Errorf("create table: %w", err) } } - vectorType := arrow.FixedSizeListOf(int32(cfg.Dims), arrow.PrimitiveTypes.Float32) - arrowSchema := arrow.NewSchema([]arrow.Field{ - {Name: "id", Type: arrow.BinaryTypes.String}, - {Name: "vector", Type: vectorType}, - }, nil) + arrowSchema, vectorType := BuildBehaviorSchema(cfg.Dims) return &LanceDBIndex{ db: db, diff --git a/third_party/sproink/include/sproink.h b/third_party/sproink/include/sproink.h index 39ba7fb8..24931c00 100644 --- a/third_party/sproink/include/sproink.h +++ b/third_party/sproink/include/sproink.h @@ -18,6 +18,11 @@ typedef struct SproinkResults SproinkResults; * Returns a heap-allocated `SproinkGraph` pointer, or null on failure. * Free with [`sproink_graph_free()`]. * + * # Resource Usage + * + * Memory scales as O(num_nodes + num_edges). For very large `num_nodes` + * values (e.g., > 10M), expect significant memory allocation. + * * # Safety * * - `sources`, `targets`, `weights`, and `kinds` must all be non-null and point @@ -38,6 +43,7 @@ struct SproinkGraph *sproink_graph_build(uint32_t num_nodes, * * - `graph` must be a pointer returned by `sproink_graph_build()`, or null. * - Must not be called more than once on the same pointer. + * Passing a previously-freed pointer is undefined behavior. */ void sproink_graph_free(struct SproinkGraph *graph); @@ -47,25 +53,45 @@ void sproink_graph_free(struct SproinkGraph *graph); * Returns a heap-allocated `SproinkResults` pointer, or null on failure. * Free with [`sproink_results_free()`]. * + * # Resource Usage + * + * - Memory: O(num_nodes) for activation arrays. For graphs above 1024 nodes, + * parallel execution allocates O(threads × 4 × num_nodes) transient memory. + * - Time: O(num_nodes × num_edges × max_steps) worst case. + * + * # Thread Safety + * + * `SproinkGraph` is immutable after construction and may be shared across + * threads. However, each `sproink_activate` call must use its own + * `SproinkResults` — results are not thread-safe. + * * # Safety * * - `graph` must be a valid pointer returned by `sproink_graph_build()`. * - `seed_nodes` and `seed_activations` must point to arrays of at least * `num_seeds` elements, or be null when `num_seeds == 0`. + * - `seed_sources` may be null (all seeds get `source: None`) or must point + * to an array of at least `num_seeds` elements. Use `u32::MAX` as the + * "no source" sentinel for individual seeds. + * - `temporal_decay_rate` and `current_time` use NaN as the "not set" + * sentinel, mapping to `None` in the engine config. */ struct SproinkResults *sproink_activate(const struct SproinkGraph *graph, uint32_t num_seeds, const uint32_t *seed_nodes, const double *seed_activations, + const uint32_t *seed_sources, uint32_t max_steps, double decay_factor, double spread_factor, double min_activation, double sigmoid_gain, double sigmoid_center, - bool inhibition_enabled, + uint8_t inhibition_enabled, double inhibition_strength, - uint32_t inhibition_breadth); + uint32_t inhibition_breadth, + double temporal_decay_rate, + double current_time); /** * Returns the number of results, or 0 if `results` is null. @@ -79,32 +105,51 @@ uint32_t sproink_results_len(const struct SproinkResults *results); /** * Copies result node IDs into `out`. * + * Writes at most `min(buffer_len, sproink_results_len(results))` elements. + * The number of elements actually written is returned (bounded by both the + * buffer capacity and the result count). + * * # Safety * * - `results` must be a valid pointer from `sproink_activate()`. - * - `out` must point to a buffer of at least `sproink_results_len(results)` elements. + * - `out` must point to a buffer of at least `buffer_len` `u32` elements, or + * be null (in which case this function is a no-op returning 0). */ -void sproink_results_nodes(const struct SproinkResults *results, uint32_t *out); +uint32_t sproink_results_nodes(const struct SproinkResults *results, + uint32_t *out, + uint32_t buffer_len); /** * Copies result activation values into `out`. * + * Writes at most `min(buffer_len, sproink_results_len(results))` elements + * and returns the count. + * * # Safety * * - `results` must be a valid pointer from `sproink_activate()`. - * - `out` must point to a buffer of at least `sproink_results_len(results)` elements. + * - `out` must point to a buffer of at least `buffer_len` `f64` elements, or + * be null (in which case this function is a no-op returning 0). */ -void sproink_results_activations(const struct SproinkResults *results, double *out); +uint32_t sproink_results_activations(const struct SproinkResults *results, + double *out, + uint32_t buffer_len); /** * Copies result hop distances into `out`. * + * Writes at most `min(buffer_len, sproink_results_len(results))` elements + * and returns the count. + * * # Safety * * - `results` must be a valid pointer from `sproink_activate()`. - * - `out` must point to a buffer of at least `sproink_results_len(results)` elements. + * - `out` must point to a buffer of at least `buffer_len` `u32` elements, or + * be null (in which case this function is a no-op returning 0). */ -void sproink_results_distances(const struct SproinkResults *results, uint32_t *out); +uint32_t sproink_results_distances(const struct SproinkResults *results, + uint32_t *out, + uint32_t buffer_len); /** * Frees results previously returned by [`sproink_activate()`]. @@ -113,6 +158,7 @@ void sproink_results_distances(const struct SproinkResults *results, uint32_t *o * * - `results` must be a pointer returned by `sproink_activate()`, or null. * - Must not be called more than once on the same pointer. + * Passing a previously-freed pointer is undefined behavior. */ void sproink_results_free(struct SproinkResults *results); @@ -145,24 +191,38 @@ uint32_t sproink_pairs_len(const struct SproinkPairs *pairs); /** * Copies pair node IDs into `out_a` and `out_b`. * + * Writes at most `min(buffer_len, sproink_pairs_len(pairs))` elements to + * each output buffer and returns the count. + * * # Safety * * - `pairs` must be a valid pointer from `sproink_extract_pairs()`. - * - `out_a` and `out_b` must each point to buffers of at least - * `sproink_pairs_len(pairs)` elements. + * - `out_a` and `out_b` must each point to buffers of at least `buffer_len` + * `u32` elements, or be null (in which case this function is a no-op + * returning 0). */ -void sproink_pairs_nodes(const struct SproinkPairs *pairs, uint32_t *out_a, uint32_t *out_b); +uint32_t sproink_pairs_nodes(const struct SproinkPairs *pairs, + uint32_t *out_a, + uint32_t *out_b, + uint32_t buffer_len); /** * Copies pair activation values into `out_a` and `out_b`. * + * Writes at most `min(buffer_len, sproink_pairs_len(pairs))` elements to + * each output buffer and returns the count. + * * # Safety * * - `pairs` must be a valid pointer from `sproink_extract_pairs()`. - * - `out_a` and `out_b` must each point to buffers of at least - * `sproink_pairs_len(pairs)` elements. + * - `out_a` and `out_b` must each point to buffers of at least `buffer_len` + * `f64` elements, or be null (in which case this function is a no-op + * returning 0). */ -void sproink_pairs_activations(const struct SproinkPairs *pairs, double *out_a, double *out_b); +uint32_t sproink_pairs_activations(const struct SproinkPairs *pairs, + double *out_a, + double *out_b, + uint32_t buffer_len); /** * Frees pairs previously returned by [`sproink_extract_pairs()`]. @@ -171,13 +231,21 @@ void sproink_pairs_activations(const struct SproinkPairs *pairs, double *out_a, * * - `pairs` must be a pointer returned by `sproink_extract_pairs()`, or null. * - Must not be called more than once on the same pointer. + * Passing a previously-freed pointer is undefined behavior. */ void sproink_pairs_free(struct SproinkPairs *pairs); /** * Computes a single Oja weight update. * - * Returns the updated weight, or `min_weight` on internal failure. + * Returns the updated weight, or `min_weight` on internal failure or invalid + * inputs. Inputs are rejected (and `min_weight` returned) if any of: + * + * - `current_weight`, `activation_a`, `activation_b`, `learning_rate`, + * `min_weight`, or `max_weight` is NaN or infinite + * - `learning_rate < 0.0` + * - `min_weight < 0.0` or `max_weight > 1.0` + * - `min_weight > max_weight` * * # Safety *