Harden desktop runtime and add multilingual memory

.github/workflows/build-go-backend.yml

@@ -5,11 +5,13 @@ on:
     branches: [main, develop]
     paths:
       - 'backend/**'
+      - 'scripts/setup-embeddings.js'
       - '.github/workflows/build-go-backend.yml'
   pull_request:
     branches: [main]
     paths:
       - 'backend/**'
+      - 'scripts/setup-embeddings.js'
       - '.github/workflows/build-go-backend.yml'
 
 jobs:
@@ -46,6 +48,11 @@ jobs:
           go-version: '1.23'
           cache-dependency-path: backend/go.sum
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
       - name: Cache Go modules
         uses: actions/cache@v4
         with:
@@ -71,6 +78,9 @@ jobs:
         working-directory: backend
         run: go mod verify
 
+      - name: Install tokenizer test artifact
+        run: node scripts/setup-embeddings.js --tokenizer-only
+
       - name: Run tests
         working-directory: backend
         env:

README.md

@@ -137,7 +137,7 @@ Follow the [Setup Instructions](https://github.com/pmbstyle/Alice/blob/main/docs
 * **Local storage**: [better-sqlite3](https://github.com/WiseLibs/better-sqlite3)
 * **Voice activity detection:** [VAD (Web)](https://github.com/ricky0123/vad)
 * **Local STT & TTS:** [whisper.cpp](https://huggingface.co/ggerganov/whisper.cpp) & [Piper](https://github.com/rhasspy/piper)
-* **Local Embeddings:** [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
+* **Local Embeddings:** [multilingual-e5-small](https://huggingface.co/intfloat/multilingual-e5-small) (ONNX, 384 dimensions)
 * **Animation:** [Kling Pro](https://fal.ai/)
 
 Other tools:
@@ -162,10 +162,13 @@ $ npm install
 Follow [setup instructions](https://github.com/pmbstyle/Alice/blob/main/docs/setupInstructions.md) to obtain required API credentials.
 
 ```bash
-# 4. Compile backend
+# 4. Download ONNX Runtime and the pinned multi-lang Memory/RAG model
+npm run setup:embeddings
+
+# 5. Compile backend
 npm run build:go
 
-# 5. Run dev environment
+# 6. Run dev environment
 $ npm run dev
 ```
 

backend/go.mod

@@ -6,6 +6,17 @@ toolchain go1.24.4
 
 require (
 	github.com/gorilla/mux v1.8.1
-	github.com/rs/cors v1.11.0
+	github.com/sugarme/tokenizer v0.3.0
 	github.com/yalue/onnxruntime_go v1.21.0
 )
+
+require (
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v2 v2.15.0 // indirect
+	github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
+	golang.org/x/text v0.25.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

backend/go.sum

@@ -1,6 +1,32 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po=
-github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
+github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
+github.com/sugarme/tokenizer v0.3.0 h1:FE8DYbNSz/kSbgEo9l/RjgYHkIJYEdskumitFQBE9FE=
+github.com/sugarme/tokenizer v0.3.0/go.mod h1:VJ+DLK5ZEZwzvODOWwY0cw+B1dabTd3nCB5HuFCItCc=
 github.com/yalue/onnxruntime_go v1.21.0 h1:DdtvfY7OP5gR8mwPDqAOAQckf+KcI30hPNJL8hQaYWI=
 github.com/yalue/onnxruntime_go v1.21.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/internal/api/embeddings.go

@@ -2,14 +2,17 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/gorilla/mux"
 )
 
 // EmbeddingRequest represents a single embedding request
 type EmbeddingRequest struct {
-	Text string `json:"text"`
+	Text      string `json:"text"`
+	InputType string `json:"input_type,omitempty"`
 }
 
 // EmbeddingResponse represents a single embedding response
@@ -20,7 +23,24 @@ type EmbeddingResponse struct {
 
 // BatchEmbeddingRequest represents a batch embedding request
 type BatchEmbeddingRequest struct {
-	Texts []string `json:"texts"`
+	Texts     []string `json:"texts"`
+	InputType string   `json:"input_type,omitempty"`
+}
+
+func prefixEmbeddingText(text, inputType string) (string, error) {
+	normalizedInputType := strings.ToLower(strings.TrimSpace(inputType))
+	if normalizedInputType != "" && normalizedInputType != "query" && normalizedInputType != "passage" {
+		return "", fmt.Errorf("unsupported input_type %q; expected query or passage", inputType)
+	}
+	if strings.HasPrefix(text, "query: ") || strings.HasPrefix(text, "passage: ") {
+		return text, nil
+	}
+	if normalizedInputType == "query" {
+		return "query: " + text, nil
+	}
+	// E5 is trained with passage prefixes for stored/indexed content. Keep
+	// this as the compatibility default for older API clients.
+	return "passage: " + text, nil
 }
 
 // BatchEmbeddingResponse represents a batch embedding response
@@ -76,7 +96,13 @@ func (h *Handler) GenerateEmbedding(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	embedding, err := embeddingService.GenerateEmbedding(r.Context(), req.Text)
+	prefixedText, err := prefixEmbeddingText(req.Text, req.InputType)
+	if err != nil {
+		h.writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
+	embedding, err := embeddingService.GenerateEmbedding(r.Context(), prefixedText)
 	if err != nil {
 		h.writeError(w, http.StatusInternalServerError, "Embedding generation failed: "+err.Error())
 		return
@@ -112,7 +138,16 @@ func (h *Handler) GenerateEmbeddings(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	embeddings, err := embeddingService.GenerateEmbeddings(r.Context(), req.Texts)
+	prefixedTexts := make([]string, len(req.Texts))
+	for i, text := range req.Texts {
+		prefixedText, err := prefixEmbeddingText(text, req.InputType)
+		if err != nil {
+			h.writeError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		prefixedTexts[i] = prefixedText
+	}
+	embeddings, err := embeddingService.GenerateEmbeddings(r.Context(), prefixedTexts)
 	if err != nil {
 		h.writeError(w, http.StatusInternalServerError, "Batch embedding generation failed: "+err.Error())
 		return

backend/internal/api/embeddings_test.go

@@ -0,0 +1,21 @@
+package api
+
+import "testing"
+
+func TestPrefixEmbeddingText(t *testing.T) {
+	if got, err := prefixEmbeddingText("как дела", "query"); err != nil || got != "query: как дела" {
+		t.Fatalf("query prefix = %q", got)
+	}
+	if got, err := prefixEmbeddingText("важный факт", "passage"); err != nil || got != "passage: важный факт" {
+		t.Fatalf("passage prefix = %q", got)
+	}
+	if got, err := prefixEmbeddingText("старый клиент", ""); err != nil || got != "passage: старый клиент" {
+		t.Fatalf("compatibility prefix = %q", got)
+	}
+	if got, err := prefixEmbeddingText("query: уже готово", "passage"); err != nil || got != "query: уже готово" {
+		t.Fatalf("existing prefix = %q", got)
+	}
+	if _, err := prefixEmbeddingText("опечатка", "qurey"); err == nil {
+		t.Fatal("expected unsupported input_type to be rejected")
+	}
+}