From cd2349044e7457dabbc46a44929900fd30833bbc Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Tue, 8 Sep 2026 19:40:10 +0300 Subject: [PATCH 01/24] feat(embeddings): add typed Admin API data-contract layer Normalize config, backfill, and capability responses so operator UI can consume a stable contract without leaking modelName or redacted secrets. --- src/lib/api/database/index.ts | 8 + src/lib/api/embeddings/index.ts | 149 ++++++++ src/lib/models/embeddings/backfill.ts | 107 ++++++ src/lib/models/embeddings/capabilities.ts | 101 ++++++ src/lib/models/embeddings/config.ts | 45 +++ src/lib/models/embeddings/errors.ts | 40 +++ src/lib/models/embeddings/index.ts | 8 + src/lib/models/embeddings/normalize.ts | 407 ++++++++++++++++++++++ src/lib/models/embeddings/search.ts | 19 + src/lib/models/embeddings/secrets.ts | 48 +++ src/lib/models/embeddings/settings.ts | 36 ++ 11 files changed, 968 insertions(+) create mode 100644 src/lib/api/embeddings/index.ts create mode 100644 src/lib/models/embeddings/backfill.ts create mode 100644 src/lib/models/embeddings/capabilities.ts create mode 100644 src/lib/models/embeddings/config.ts create mode 100644 src/lib/models/embeddings/errors.ts create mode 100644 src/lib/models/embeddings/index.ts create mode 100644 src/lib/models/embeddings/normalize.ts create mode 100644 src/lib/models/embeddings/search.ts create mode 100644 src/lib/models/embeddings/secrets.ts create mode 100644 src/lib/models/embeddings/settings.ts diff --git a/src/lib/api/database/index.ts b/src/lib/api/database/index.ts index 5eb9343a5..9e81c61fd 100644 --- a/src/lib/api/database/index.ts +++ b/src/lib/api/database/index.ts @@ -11,6 +11,7 @@ import { SchemaOptions, } from '@/lib/models/database'; import { CustomEndpoint } from '@/lib/models/database/custom-endpoints'; +import { unwrapVectorIndexes } from '@/lib/models/embeddings'; export const getPendingSchemas = async (args: { skip?: number; @@ -353,6 +354,13 @@ export const getSchemaIndexes = async (schemaId: string) => { .then(res => res.data); }; +export const getSchemaVectorIndexes = async (schemaId: string) => { + const res = await ( + await getApiClient() + ).get(`/database/schemas/${schemaId}/vector-indexes`); + return unwrapVectorIndexes(res.data); +}; + export const deleteSchemaIndexes = async ( schemaId: string, indexNames: string[] diff --git a/src/lib/api/embeddings/index.ts b/src/lib/api/embeddings/index.ts new file mode 100644 index 000000000..eba11a467 --- /dev/null +++ b/src/lib/api/embeddings/index.ts @@ -0,0 +1,149 @@ +'use server'; + +import { getApiClient } from '@/lib/api'; +import { afterPatchServing } from '@/lib/api/modules/afterPatchServing'; +import { PatchSettingsOptions } from '@/lib/api/modules/patch-settings-options'; +import { + BackfillListQuery, + EmbeddingConfig, + EmbeddingConfigInput, + EmbeddingsConfigResponse, + EmbeddingsSettings, + sanitizeEmbeddingsSettingsPatch, + SemanticSearchInput, + serializeBackfillFilter, + StartBackfillInput, + unwrapBackfillList, + unwrapBackfillRun, + unwrapDeletedEmbeddingConfig, + unwrapEmbeddingConfig, + unwrapEmbeddingConfigList, + unwrapEmbeddingsCapabilities, + unwrapEmbeddingsSettings, + unwrapEmbeddingsStatus, + unwrapSemanticSearch, + unwrapStartBackfill, + unwrapUpsertEmbeddingConfig, +} from '@/lib/models/embeddings'; + +export const getEmbeddingConfigs = async (args?: { + schemaName?: string; + id?: string; +}) => { + const res = await ( + await getApiClient() + ).get('/embeddings/configs', { params: args }); + return unwrapEmbeddingConfigList(res.data); +}; + +export const getEmbeddingConfig = async (id: string) => { + const res = await ( + await getApiClient() + ).get(`/embeddings/configs/${id}`); + return unwrapEmbeddingConfig(res.data); +}; + +export const upsertEmbeddingConfig = async (data: EmbeddingConfigInput) => { + const res = await ( + await getApiClient() + ).post('/embeddings/configs', data); + return unwrapUpsertEmbeddingConfig(res.data); +}; + +export const deleteEmbeddingConfig = async ( + id: string +): Promise => { + const res = await ( + await getApiClient() + ).delete(`/embeddings/configs/${id}`); + return unwrapDeletedEmbeddingConfig(res.data); +}; + +export const getEmbeddingsCapabilities = async (schemaName?: string) => { + const res = await ( + await getApiClient() + ).get('/embeddings/capabilities', { + params: schemaName ? { schemaName } : undefined, + }); + return unwrapEmbeddingsCapabilities(res.data); +}; + +export const getEmbeddingsStatus = async (schemaName?: string) => { + const res = await ( + await getApiClient() + ).get('/embeddings/status', { + params: schemaName ? { schemaName } : undefined, + }); + return unwrapEmbeddingsStatus(res.data); +}; + +export const getBackfills = async (args?: BackfillListQuery) => { + const res = await ( + await getApiClient() + ).get('/embeddings/backfills', { params: args }); + return unwrapBackfillList(res.data); +}; + +export const getBackfill = async (id: string) => { + const res = await ( + await getApiClient() + ).get(`/embeddings/backfills/${id}`); + return unwrapBackfillRun(res.data); +}; + +export const startBackfill = async (data: StartBackfillInput) => { + const filter = serializeBackfillFilter(data.filter); + const res = await ( + await getApiClient() + ).post('/embeddings/backfills', { + schemaName: data.schemaName, + batchSize: data.batchSize, + configId: data.configId, + onlyMissing: data.onlyMissing, + ...(filter ? { filter } : {}), + }); + return unwrapStartBackfill(res.data); +}; + +export const cancelBackfill = async (id: string) => { + const res = await ( + await getApiClient() + ).post(`/embeddings/backfills/${id}/cancel`); + return unwrapBackfillRun(res.data); +}; + +export const resumeBackfill = async (id: string) => { + const res = await ( + await getApiClient() + ).post(`/embeddings/backfills/${id}/resume`); + return unwrapBackfillRun(res.data); +}; + +export const searchEmbeddings = async (data: SemanticSearchInput) => { + const filter = serializeBackfillFilter(data.filter); + const res = await ( + await getApiClient() + ).post('/embeddings/search', { + schemaName: data.schemaName, + text: data.text, + targetField: data.targetField, + limit: data.limit, + ...(filter ? { filter } : {}), + }); + return unwrapSemanticSearch(res.data); +}; + +export const getEmbeddingsSettings = + async (): Promise => { + const res = await (await getApiClient()).get('/config/embeddings'); + return unwrapEmbeddingsSettings(res.data); + }; + +export const patchEmbeddingsSettings = async ( + data: Partial, + options?: PatchSettingsOptions +) => { + const config = sanitizeEmbeddingsSettingsPatch(data); + await (await getApiClient()).patch('/config/embeddings', { config }); + return afterPatchServing(options); +}; diff --git a/src/lib/models/embeddings/backfill.ts b/src/lib/models/embeddings/backfill.ts new file mode 100644 index 000000000..fd6cdd9b0 --- /dev/null +++ b/src/lib/models/embeddings/backfill.ts @@ -0,0 +1,107 @@ +export const BACKFILL_RUN_STATES = [ + 'queued', + 'running', + 'completed', + 'failed', + 'canceled', +] as const; + +export type BackfillRunState = (typeof BACKFILL_RUN_STATES)[number]; + +export const ACTIVE_BACKFILL_STATES: readonly BackfillRunState[] = [ + 'queued', + 'running', +]; + +export type BackfillFilter = Record; + +export type BackfillRun = { + _id: string; + schemaName: string; + configId?: string; + state: BackfillRunState; + cursor?: string; + batchSize: number; + onlyMissing: boolean; + filter?: BackfillFilter; + scannedCount: number; + queuedCount: number; + processedCount: number; + failedCount: number; + startedAt?: string; + finishedAt?: string; + drainStartedAt?: string; + error?: string; + createdAt?: string; + updatedAt?: string; +}; + +export type BackfillListQuery = { + schemaName?: string; + state?: BackfillRunState; + configId?: string; + skip?: number; + limit?: number; +}; + +export type BackfillListResponse = { + runs: BackfillRun[]; + count: number; +}; + +export type StartBackfillInput = { + schemaName: string; + batchSize?: number; + configId?: string; + onlyMissing?: boolean; + filter?: BackfillFilter | string; +}; + +export type StartBackfillResult = { + queued: number; + runs: BackfillRun[]; + warnings: string[]; +}; + +export function isBackfillRunState(value: unknown): value is BackfillRunState { + return ( + typeof value === 'string' && + (BACKFILL_RUN_STATES as readonly string[]).includes(value) + ); +} + +export function isActiveBackfillState(state: BackfillRunState): boolean { + return ACTIVE_BACKFILL_STATES.includes(state); +} + +export function canCancelBackfill(state: BackfillRunState): boolean { + switch (state) { + case 'queued': + case 'running': + return true; + case 'completed': + case 'failed': + case 'canceled': + return false; + default: { + const exhaustive: never = state; + return exhaustive; + } + } +} + +export function isResumeEligible(state: BackfillRunState): boolean { + switch (state) { + case 'failed': + case 'canceled': + return true; + case 'queued': + case 'running': + case 'completed': + return false; + default: { + const exhaustive: never = state; + return exhaustive; + } + } +} diff --git a/src/lib/models/embeddings/capabilities.ts b/src/lib/models/embeddings/capabilities.ts new file mode 100644 index 000000000..2286b789c --- /dev/null +++ b/src/lib/models/embeddings/capabilities.ts @@ -0,0 +1,101 @@ +import { VectorSimilarity } from '@/lib/models/embeddings/config'; + +export const VECTOR_INDEX_STATUSES = ['pending', 'ready', 'failed'] as const; + +export type VectorIndexStatus = (typeof VECTOR_INDEX_STATUSES)[number]; + +export const VECTOR_INDEX_METHODS = ['hnsw', 'ivfflat', 'flat'] as const; + +export type VectorIndexMethod = (typeof VECTOR_INDEX_METHODS)[number]; + +export const VECTOR_CAPABILITY_PROVIDERS = [ + 'mongodb', + 'postgres', + 'unsupported', +] as const; + +export type VectorCapabilityProvider = + (typeof VECTOR_CAPABILITY_PROVIDERS)[number]; + +export type VectorIndexDefinition = { + name?: string; + field: string; + dimensions: number; + similarity: VectorSimilarity; + method?: VectorIndexMethod; + filterFields?: string[]; + status?: VectorIndexStatus; + queryable?: boolean; +}; + +export type SchemaVectorIndexesResponse = { + indexes: VectorIndexDefinition[]; +}; + +export type VectorCapabilities = { + supported: boolean; + storage: boolean; + indexing: boolean; + search: boolean; + provider: VectorCapabilityProvider; + reason?: string; +}; + +export type EmbeddingsCapabilitiesResponse = { + capabilities: VectorCapabilities; + warnings: string[]; +}; + +export type EmbeddingsQueueCounts = { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; +}; + +export type EmbeddingsStatus = { + enabled: boolean; + ready: boolean; + capabilities: VectorCapabilities; + generationQueue: EmbeddingsQueueCounts; + backfillQueue: EmbeddingsQueueCounts; + warnings: string[]; +}; + +export function isVectorIndexStatus( + value: unknown +): value is VectorIndexStatus { + return ( + typeof value === 'string' && + (VECTOR_INDEX_STATUSES as readonly string[]).includes(value) + ); +} + +export function isVectorIndexMethod( + value: unknown +): value is VectorIndexMethod { + return ( + typeof value === 'string' && + (VECTOR_INDEX_METHODS as readonly string[]).includes(value) + ); +} + +export function isVectorCapabilityProvider( + value: unknown +): value is VectorCapabilityProvider { + return ( + typeof value === 'string' && + (VECTOR_CAPABILITY_PROVIDERS as readonly string[]).includes(value) + ); +} + +export function isVectorIndexQueryable(index?: VectorIndexDefinition): boolean { + if (!index) return false; + if (index.queryable === false) return false; + if (index.status === 'failed') return false; + if (index.status === 'pending' && index.queryable !== true) return false; + if (index.queryable === true) return true; + return index.status === 'ready'; +} diff --git a/src/lib/models/embeddings/config.ts b/src/lib/models/embeddings/config.ts new file mode 100644 index 000000000..f07336b8f --- /dev/null +++ b/src/lib/models/embeddings/config.ts @@ -0,0 +1,45 @@ +export const VECTOR_SIMILARITIES = [ + 'cosine', + 'euclidean', + 'dotProduct', +] as const; + +export type VectorSimilarity = (typeof VECTOR_SIMILARITIES)[number]; + +export type EmbeddingConfig = { + _id: string; + schemaName: string; + sourceFields: string[]; + targetField: string; + provider: string; + model: string; + dimensions: number; + similarity: VectorSimilarity; + enabled: boolean; + createdAt?: string; + updatedAt?: string; +}; + +export type EmbeddingConfigInput = { + schemaName: string; + sourceFields: string[]; + targetField: string; + provider?: string; + model?: string; + dimensions: number; + similarity?: VectorSimilarity; + sourceFieldAllowlist?: string[]; + enabled?: boolean; +}; + +export type UpsertEmbeddingConfigResult = { + config: EmbeddingConfig; + warnings: string[]; +}; + +export function isVectorSimilarity(value: unknown): value is VectorSimilarity { + return ( + typeof value === 'string' && + (VECTOR_SIMILARITIES as readonly string[]).includes(value) + ); +} diff --git a/src/lib/models/embeddings/errors.ts b/src/lib/models/embeddings/errors.ts new file mode 100644 index 000000000..e319b1736 --- /dev/null +++ b/src/lib/models/embeddings/errors.ts @@ -0,0 +1,40 @@ +type AxiosLikeError = { + response?: { + status?: number; + data?: unknown; + }; + message?: string; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isAxiosLikeError(err: unknown): err is AxiosLikeError { + return Boolean(err && typeof err === 'object' && 'response' in err); +} + +function readBackendMessage(data: unknown): string | undefined { + if (typeof data === 'string' && data.trim().length > 0) return data; + if (!isRecord(data)) return undefined; + if (typeof data.message === 'string' && data.message.trim().length > 0) { + return data.message; + } + if (typeof data.error === 'string' && data.error.trim().length > 0) { + return data.error; + } + if (isRecord(data.error) && typeof data.error.message === 'string') { + const nested = data.error.message.trim(); + if (nested.length > 0) return nested; + } + return undefined; +} + +export function formatEmbeddingsApiError(err: unknown): string { + if (isAxiosLikeError(err)) { + return ( + readBackendMessage(err.response?.data) ?? err.message ?? 'Request failed' + ); + } + return err instanceof Error ? err.message : 'Request failed'; +} diff --git a/src/lib/models/embeddings/index.ts b/src/lib/models/embeddings/index.ts new file mode 100644 index 000000000..d84532a3d --- /dev/null +++ b/src/lib/models/embeddings/index.ts @@ -0,0 +1,8 @@ +export * from './config'; +export * from './backfill'; +export * from './capabilities'; +export * from './search'; +export * from './settings'; +export * from './errors'; +export * from './secrets'; +export * from './normalize'; diff --git a/src/lib/models/embeddings/normalize.ts b/src/lib/models/embeddings/normalize.ts new file mode 100644 index 000000000..a4f2182a3 --- /dev/null +++ b/src/lib/models/embeddings/normalize.ts @@ -0,0 +1,407 @@ +import { + BackfillListResponse, + BackfillRun, + BackfillRunState, + isBackfillRunState, + StartBackfillResult, +} from '@/lib/models/embeddings/backfill'; +import { + EmbeddingsCapabilitiesResponse, + EmbeddingsQueueCounts, + EmbeddingsStatus, + isVectorCapabilityProvider, + isVectorIndexMethod, + isVectorIndexStatus, + SchemaVectorIndexesResponse, + VectorCapabilities, + VectorIndexDefinition, +} from '@/lib/models/embeddings/capabilities'; +import { + EmbeddingConfig, + isVectorSimilarity, + UpsertEmbeddingConfigResult, +} from '@/lib/models/embeddings/config'; +import { + SemanticSearchHit, + SemanticSearchResponse, +} from '@/lib/models/embeddings/search'; +import { + EmbeddingsConfigResponse, + EmbeddingsProviderSettings, + EmbeddingsQueueSettings, + EmbeddingsSecuritySettings, + EmbeddingsSettings, + OPENAI_COMPATIBLE_PROVIDER, +} from '@/lib/models/embeddings/settings'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function readString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function readId(value: Record): string | undefined { + return readString(value._id) ?? readString(value.id); +} + +function readNumber(value: unknown, fallback = 0): number { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value === 'string' && value.trim() !== '') { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return fallback; +} + +function readBoolean(value: unknown, fallback = false): boolean { + return typeof value === 'boolean' ? value : fallback; +} + +function readStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === 'string'); +} + +export function parseBackfillFilter( + value: unknown +): Record | undefined { + if (value == null || value === '') return undefined; + if (typeof value === 'string') { + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } + } + if (isRecord(value)) return { ...value }; + return undefined; +} + +export function serializeBackfillFilter( + filter: Record | string | undefined +): Record | undefined { + if (filter == null || filter === '') return undefined; + const parsed = parseBackfillFilter(filter); + if (!parsed) { + throw new Error('filter must be a JSON object'); + } + return parsed; +} + +function unwrapNamedRecord( + payload: unknown, + key: 'config' | 'run' +): Record { + if (!isRecord(payload)) { + throw new Error('Invalid embeddings response'); + } + const nested = payload[key]; + if (isRecord(nested) && (readId(nested) || readString(nested.schemaName))) { + if ( + readString(payload.schemaName) && + (Array.isArray(payload.sourceFields) || typeof payload.state === 'string') + ) { + return payload; + } + return nested; + } + return payload; +} + +export function unwrapEmbeddingConfig(payload: unknown): EmbeddingConfig { + const raw = unwrapNamedRecord(payload, 'config'); + const _id = readId(raw); + const schemaName = readString(raw.schemaName); + const targetField = readString(raw.targetField); + if (!_id || !schemaName || !targetField) { + throw new Error('Invalid embedding config response'); + } + return { + _id, + schemaName, + sourceFields: readStringArray(raw.sourceFields), + targetField, + provider: readString(raw.provider) ?? '', + model: readString(raw.model) ?? readString(raw.modelName) ?? '', + dimensions: readNumber(raw.dimensions), + similarity: isVectorSimilarity(raw.similarity) ? raw.similarity : 'cosine', + enabled: raw.enabled !== false, + createdAt: readString(raw.createdAt), + updatedAt: readString(raw.updatedAt), + }; +} + +export function unwrapEmbeddingConfigList(payload: unknown): EmbeddingConfig[] { + if (Array.isArray(payload)) { + return payload.map(unwrapEmbeddingConfig); + } + if (isRecord(payload) && Array.isArray(payload.configs)) { + return payload.configs.map(unwrapEmbeddingConfig); + } + if (isRecord(payload) && isRecord(payload.config)) { + return [unwrapEmbeddingConfig(payload.config)]; + } + return []; +} + +export function unwrapUpsertEmbeddingConfig( + payload: unknown +): UpsertEmbeddingConfigResult { + if (!isRecord(payload)) { + throw new Error('Invalid embedding config response'); + } + return { + config: unwrapEmbeddingConfig( + isRecord(payload.config) ? payload.config : payload + ), + warnings: readStringArray(payload.warnings), + }; +} + +export function unwrapDeletedEmbeddingConfig( + payload: unknown +): EmbeddingConfig { + return unwrapEmbeddingConfig(payload); +} + +export function unwrapBackfillRun(payload: unknown): BackfillRun { + const raw = unwrapNamedRecord(payload, 'run'); + const _id = readId(raw); + const schemaName = readString(raw.schemaName); + const state: BackfillRunState | undefined = isBackfillRunState(raw.state) + ? raw.state + : undefined; + if (!_id || !schemaName || !state) { + throw new Error('Invalid backfill response'); + } + return { + _id, + schemaName, + configId: readString(raw.configId), + state, + cursor: readString(raw.cursor), + batchSize: readNumber(raw.batchSize, 100), + onlyMissing: readBoolean(raw.onlyMissing), + filter: parseBackfillFilter(raw.filter), + scannedCount: readNumber(raw.scannedCount), + queuedCount: readNumber(raw.queuedCount), + processedCount: readNumber(raw.processedCount), + failedCount: readNumber(raw.failedCount), + startedAt: readString(raw.startedAt), + finishedAt: readString(raw.finishedAt), + drainStartedAt: readString(raw.drainStartedAt), + error: readString(raw.error), + createdAt: readString(raw.createdAt), + updatedAt: readString(raw.updatedAt), + }; +} + +export function unwrapBackfillList(payload: unknown): BackfillListResponse { + if (!isRecord(payload) || !Array.isArray(payload.runs)) { + throw new Error('Invalid backfill list response'); + } + return { + runs: payload.runs.map(unwrapBackfillRun), + count: readNumber(payload.count, payload.runs.length), + }; +} + +export function unwrapStartBackfill(payload: unknown): StartBackfillResult { + if (!isRecord(payload)) { + throw new Error('Invalid backfill start response'); + } + const runs = Array.isArray(payload.runs) + ? payload.runs.map(unwrapBackfillRun) + : []; + return { + queued: readNumber(payload.queued, runs.length), + runs, + warnings: readStringArray(payload.warnings), + }; +} + +export function unwrapEmbeddingsCapabilities( + payload: unknown +): EmbeddingsCapabilitiesResponse { + if (!isRecord(payload) || !isRecord(payload.capabilities)) { + throw new Error('Invalid embeddings capabilities response'); + } + return { + capabilities: unwrapDatabaseVectorCapabilities(payload.capabilities), + warnings: readStringArray(payload.warnings), + }; +} + +export function unwrapDatabaseVectorCapabilities( + payload: unknown +): VectorCapabilities { + if (!isRecord(payload) || typeof payload.supported !== 'boolean') { + throw new Error('Invalid database vector capabilities response'); + } + return { + supported: payload.supported, + storage: readBoolean(payload.storage), + indexing: readBoolean(payload.indexing), + search: readBoolean(payload.search), + provider: isVectorCapabilityProvider(payload.provider) + ? payload.provider + : 'unsupported', + reason: readString(payload.reason), + }; +} + +function unwrapQueueCounts(value: unknown): EmbeddingsQueueCounts { + const raw = isRecord(value) ? value : {}; + return { + waiting: readNumber(raw.waiting), + active: readNumber(raw.active), + completed: readNumber(raw.completed), + failed: readNumber(raw.failed), + delayed: readNumber(raw.delayed), + paused: readNumber(raw.paused), + }; +} + +export function unwrapEmbeddingsStatus(payload: unknown): EmbeddingsStatus { + if (!isRecord(payload) || !isRecord(payload.capabilities)) { + throw new Error('Invalid embeddings status response'); + } + return { + enabled: readBoolean(payload.enabled), + ready: readBoolean(payload.ready), + capabilities: unwrapDatabaseVectorCapabilities(payload.capabilities), + generationQueue: unwrapQueueCounts(payload.generationQueue), + backfillQueue: unwrapQueueCounts(payload.backfillQueue), + warnings: readStringArray(payload.warnings), + }; +} + +export function unwrapVectorIndexes( + payload: unknown +): SchemaVectorIndexesResponse { + const indexes = Array.isArray(payload) + ? payload + : isRecord(payload) && Array.isArray(payload.indexes) + ? payload.indexes + : []; + return { indexes: indexes.map(unwrapVectorIndex) }; +} + +function unwrapVectorIndex(value: unknown): VectorIndexDefinition { + if (!isRecord(value) || typeof value.field !== 'string') { + throw new Error('Invalid vector index response'); + } + return { + name: readString(value.name), + field: value.field, + dimensions: readNumber(value.dimensions), + similarity: isVectorSimilarity(value.similarity) + ? value.similarity + : 'cosine', + method: isVectorIndexMethod(value.method) ? value.method : undefined, + filterFields: Array.isArray(value.filterFields) + ? readStringArray(value.filterFields) + : undefined, + status: isVectorIndexStatus(value.status) ? value.status : undefined, + queryable: + typeof value.queryable === 'boolean' ? value.queryable : undefined, + }; +} + +function unwrapSearchDocument(value: unknown): Record { + if (isRecord(value)) return value; + if (typeof value !== 'string' || value.length === 0) return {}; + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) ? parsed : {}; + } catch { + return {}; + } +} + +export function unwrapSemanticSearch(payload: unknown): SemanticSearchResponse { + const hits = Array.isArray(payload) + ? payload + : isRecord(payload) && Array.isArray(payload.hits) + ? payload.hits + : undefined; + if (!hits) { + throw new Error('Invalid semantic search response'); + } + return { hits: hits.map(unwrapSearchHit) }; +} + +function unwrapSearchHit(value: unknown): SemanticSearchHit { + if (!isRecord(value) || typeof value.score !== 'number') { + throw new Error('Invalid semantic search hit'); + } + return { + document: unwrapSearchDocument(value.document), + score: value.score, + distance: typeof value.distance === 'number' ? value.distance : undefined, + metric: readString(value.metric), + provider: readString(value.provider), + }; +} + +function unwrapProviderSettings(value: unknown): EmbeddingsProviderSettings { + const raw = isRecord(value) ? value : {}; + return { + endpoint: typeof raw.endpoint === 'string' ? raw.endpoint : '', + apiKey: typeof raw.apiKey === 'string' ? raw.apiKey : undefined, + model: typeof raw.model === 'string' ? raw.model : '', + allowedHosts: readStringArray(raw.allowedHosts), + }; +} + +function unwrapQueueSettings(value: unknown): EmbeddingsQueueSettings { + const raw = isRecord(value) ? value : {}; + return { + concurrency: readNumber(raw.concurrency, 2), + attempts: readNumber(raw.attempts, 3), + maxBatchSize: readNumber(raw.maxBatchSize, 500), + drainTimeoutMs: readNumber(raw.drainTimeoutMs, 15 * 60 * 1000), + }; +} + +function unwrapSecuritySettings(value: unknown): EmbeddingsSecuritySettings { + const raw = isRecord(value) ? value : {}; + return { + requireGrpcKey: readBoolean(raw.requireGrpcKey), + sourceFieldAllowlist: readStringArray(raw.sourceFieldAllowlist), + maxMutationEventIds: readNumber(raw.maxMutationEventIds, 500), + embedTimeoutMs: readNumber(raw.embedTimeoutMs, 10_000), + maxEmbedInputBytes: readNumber(raw.maxEmbedInputBytes, 32 * 1024), + maxEmbedResponseBytes: readNumber(raw.maxEmbedResponseBytes, 1024 * 1024), + }; +} + +export function unwrapEmbeddingsSettings( + payload: unknown +): EmbeddingsConfigResponse { + if (!isRecord(payload)) { + throw new Error('Invalid embeddings settings response'); + } + const raw = isRecord(payload.config) ? payload.config : payload; + const providersRaw = isRecord(raw.providers) ? raw.providers : {}; + const providers: Record = {}; + for (const [name, provider] of Object.entries(providersRaw)) { + providers[name] = unwrapProviderSettings(provider); + } + if (!providers[OPENAI_COMPATIBLE_PROVIDER]) { + providers[OPENAI_COMPATIBLE_PROVIDER] = unwrapProviderSettings(undefined); + } + return { + config: { + enabled: readBoolean(raw.enabled), + defaultProvider: + readString(raw.defaultProvider) ?? OPENAI_COMPATIBLE_PROVIDER, + providers, + queue: unwrapQueueSettings(raw.queue), + security: unwrapSecuritySettings(raw.security), + }, + }; +} diff --git a/src/lib/models/embeddings/search.ts b/src/lib/models/embeddings/search.ts new file mode 100644 index 000000000..903950abf --- /dev/null +++ b/src/lib/models/embeddings/search.ts @@ -0,0 +1,19 @@ +export type SemanticSearchInput = { + schemaName: string; + text: string; + targetField?: string; + filter?: Record | string; + limit?: number; +}; + +export type SemanticSearchHit = { + document: Record; + score: number; + distance?: number; + metric?: string; + provider?: string; +}; + +export type SemanticSearchResponse = { + hits: SemanticSearchHit[]; +}; diff --git a/src/lib/models/embeddings/secrets.ts b/src/lib/models/embeddings/secrets.ts new file mode 100644 index 000000000..41e9aede9 --- /dev/null +++ b/src/lib/models/embeddings/secrets.ts @@ -0,0 +1,48 @@ +import { + EmbeddingsProviderSettings, + EmbeddingsSettings, +} from '@/lib/models/embeddings/settings'; + +export const REDACTED_SECRET = '[REDACTED]'; + +export function isSecretConfigured(value: string | undefined): boolean { + return typeof value === 'string' && value.length > 0; +} + +export function isRedactedSecret(value: string | undefined): boolean { + return value === REDACTED_SECRET; +} + +export function shouldSubmitApiKey(value: string | undefined): value is string { + return isSecretConfigured(value) && !isRedactedSecret(value); +} + +export function omitRedactedApiKey( + provider: EmbeddingsProviderSettings +): EmbeddingsProviderSettings { + if (!shouldSubmitApiKey(provider.apiKey)) { + return { + endpoint: provider.endpoint, + model: provider.model, + allowedHosts: provider.allowedHosts, + }; + } + return { + endpoint: provider.endpoint, + apiKey: provider.apiKey, + model: provider.model, + allowedHosts: provider.allowedHosts, + }; +} + +export function sanitizeEmbeddingsSettingsPatch( + data: Partial +): Partial { + if (!data.providers) return data; + const providers: Record = {}; + for (const [name, provider] of Object.entries(data.providers)) { + if (!provider) continue; + providers[name] = omitRedactedApiKey(provider); + } + return { ...data, providers }; +} diff --git a/src/lib/models/embeddings/settings.ts b/src/lib/models/embeddings/settings.ts new file mode 100644 index 000000000..b78e59d4e --- /dev/null +++ b/src/lib/models/embeddings/settings.ts @@ -0,0 +1,36 @@ +export const OPENAI_COMPATIBLE_PROVIDER = 'openai-compatible' as const; + +export type EmbeddingsProviderSettings = { + endpoint: string; + apiKey?: string; + model: string; + allowedHosts: string[]; +}; + +export type EmbeddingsQueueSettings = { + concurrency: number; + attempts: number; + maxBatchSize: number; + drainTimeoutMs: number; +}; + +export type EmbeddingsSecuritySettings = { + requireGrpcKey: boolean; + sourceFieldAllowlist: string[]; + maxMutationEventIds: number; + embedTimeoutMs: number; + maxEmbedInputBytes: number; + maxEmbedResponseBytes: number; +}; + +export type EmbeddingsSettings = { + enabled: boolean; + defaultProvider: string; + providers: Record; + queue: EmbeddingsQueueSettings; + security: EmbeddingsSecuritySettings; +}; + +export type EmbeddingsConfigResponse = { + config: EmbeddingsSettings; +}; From a58c888c27db5df89ddffbf609e098047290409b Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Tue, 8 Sep 2026 19:49:06 +0300 Subject: [PATCH 02/24] feat(embeddings): register module shell and readiness overview Add navigation, maps, and a fail-soft overview with a reusable capability, provider, index, config, and worker checklist. --- .../(dashboard)/(modules)/embeddings/page.tsx | 372 ++++++++++++++++++ src/app/(dashboard)/template.tsx | 31 +- src/components/dashboard/ActiveModuleGrid.tsx | 2 + .../embeddings/EmbeddingsReadiness.tsx | 110 ++++++ src/components/navigation/navList.config.ts | 12 + src/lib/models/embeddings/index.ts | 1 + src/lib/models/embeddings/readiness.ts | 292 ++++++++++++++ src/lib/models/logs-viewer/constants.tsx | 1 + src/lib/prometheus/metrics.ts | 2 + src/lib/utils/module-utils.ts | 3 + 10 files changed, 821 insertions(+), 5 deletions(-) create mode 100644 src/app/(dashboard)/(modules)/embeddings/page.tsx create mode 100644 src/components/embeddings/EmbeddingsReadiness.tsx create mode 100644 src/lib/models/embeddings/readiness.ts diff --git a/src/app/(dashboard)/(modules)/embeddings/page.tsx b/src/app/(dashboard)/(modules)/embeddings/page.tsx new file mode 100644 index 000000000..2b0bd1ad2 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/page.tsx @@ -0,0 +1,372 @@ +import { + AlertTriangle, + Info, + Plus, + ScanSearch, + Search, + Settings, +} from 'lucide-react'; +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { ModuleDashboard } from '@/components/dashboard/ModuleDashboard'; +import { MetricCardProps } from '@/components/dashboard/MetricCard'; +import { ModuleStatus } from '@/components/dashboard/ModuleStatusCard'; +import { QuickAction } from '@/components/dashboard/QuickActionsCard'; +import { EmbeddingsReadiness } from '@/components/embeddings/EmbeddingsReadiness'; +import { + getBackfills, + getEmbeddingConfigs, + getEmbeddingsCapabilities, + getEmbeddingsSettings, + getEmbeddingsStatus, +} from '@/lib/api/embeddings'; +import { getSchemas, getSchemaVectorIndexes } from '@/lib/api/database'; +import { + deriveEmbeddingsReadiness, + EmbeddingConfig, + EmbeddingsQueueCounts, + EmbeddingsStatus, + formatEmbeddingsApiError, + SchemaIndexLookup, +} from '@/lib/models/embeddings'; +import { getPrometheusAvailability } from '@/lib/observability/prometheusAvailability'; +import { + getModuleStatus, + getModuleUptime, + getSystemMetrics, +} from '@/lib/prometheus/metrics'; +import { getApiModuleNameFromPath } from '@/lib/utils/module-utils'; + +function settledValue(result: PromiseSettledResult): T | undefined { + return result.status === 'fulfilled' ? result.value : undefined; +} + +function settledError( + result: PromiseSettledResult +): string | undefined { + if (result.status !== 'rejected') return undefined; + return formatEmbeddingsApiError(result.reason); +} + +function emptyQueue(): EmbeddingsQueueCounts { + return { + waiting: 0, + active: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, + }; +} + +function addQueues( + left: EmbeddingsQueueCounts, + right: EmbeddingsQueueCounts +): EmbeddingsQueueCounts { + return { + waiting: left.waiting + right.waiting, + active: left.active + right.active, + completed: left.completed + right.completed, + failed: left.failed + right.failed, + delayed: left.delayed + right.delayed, + paused: left.paused + right.paused, + }; +} + +async function resolveIndexesBySchema( + configs: EmbeddingConfig[] +): Promise> { + const schemaNames = [...new Set(configs.map(config => config.schemaName))]; + if (schemaNames.length === 0) return {}; + + let schemas: { name: string; _id: string }[] | undefined; + try { + const response = await getSchemas({ limit: 1000 }); + schemas = response.schemas; + } catch { + return Object.fromEntries(schemaNames.map(name => [name, 'unknown'])); + } + + const idByName = new Map(schemas.map(schema => [schema.name, schema._id])); + const lookups = await Promise.allSettled( + schemaNames.map(async (name): Promise<[string, SchemaIndexLookup]> => { + const schemaId = idByName.get(name); + if (!schemaId) return [name, []]; + const { indexes } = await getSchemaVectorIndexes(schemaId); + return [name, indexes]; + }) + ); + + const map: Record = {}; + for (const [index, name] of schemaNames.entries()) { + const result = lookups[index]; + map[name] = result.status === 'fulfilled' ? result.value[1] : 'unknown'; + } + return map; +} + +function metricValue( + value: number | undefined, + unavailable: boolean +): { value: string; status: MetricCardProps['status'] } { + if (unavailable) { + return { value: '—', status: 'unknown' }; + } + return { + value: (value ?? 0).toLocaleString(), + status: 'healthy', + }; +} + +export default async function EmbeddingsDashboard() { + const apiModuleName = getApiModuleNameFromPath('/embeddings') || 'embeddings'; + + const [ + statusResult, + capabilitiesResult, + configsResult, + settingsResult, + backfillsResult, + promAvailabilityResult, + moduleHealthResult, + uptimeResult, + systemMetricsResult, + ] = await Promise.allSettled([ + getEmbeddingsStatus(), + getEmbeddingsCapabilities(), + getEmbeddingConfigs(), + getEmbeddingsSettings(), + getBackfills({ skip: 0, limit: 1 }), + getPrometheusAvailability(), + getModuleStatus(apiModuleName), + getModuleUptime(apiModuleName), + getSystemMetrics(apiModuleName), + ]); + + const status = settledValue(statusResult); + const capabilities = + settledValue(capabilitiesResult)?.capabilities ?? status?.capabilities; + const configs = settledValue(configsResult); + const settings = settledValue(settingsResult)?.config; + const backfills = settledValue(backfillsResult); + const promAvailability = settledValue(promAvailabilityResult); + + const indexesBySchema = configs + ? await resolveIndexesBySchema(configs) + : undefined; + + const workersEnabled = status?.enabled ?? settings?.enabled; + const rows = deriveEmbeddingsReadiness({ + capabilities, + capabilitiesError: settledError(capabilitiesResult), + settings, + settingsError: settledError(settingsResult), + configs, + configsError: settledError(configsResult), + indexesBySchema, + workersEnabled, + }); + + const queueUnavailable = !status; + const queues = status + ? addQueues(status.generationQueue, status.backfillQueue) + : emptyQueue(); + + const metricCards: MetricCardProps[] = [ + { + title: 'Queue waiting', + ...metricValue(queues.waiting, queueUnavailable), + description: 'Generation and backfill jobs waiting', + status: queueUnavailable + ? 'unknown' + : queues.waiting > 0 + ? 'warning' + : 'healthy', + }, + { + title: 'Queue active', + ...metricValue(queues.active, queueUnavailable), + description: 'Jobs currently processing', + }, + { + title: 'Queue failed', + ...metricValue(queues.failed, queueUnavailable), + description: 'Failed generation and backfill jobs', + status: queueUnavailable + ? 'unknown' + : queues.failed > 0 + ? 'critical' + : 'healthy', + }, + { + title: 'Configs', + ...metricValue(configs?.length, configs == null), + description: + backfills == null + ? 'Embedding configurations' + : `${backfills.count.toLocaleString()} backfill runs`, + }, + ]; + + const moduleStatus: ModuleStatus = { + name: 'Embeddings', + status: settledValue(moduleHealthResult) ?? 'unknown', + uptime: settledValue(uptimeResult) ?? 'Unknown', + version: '1.0.0', + instances: 1, + description: 'Vector embeddings, indexes, and semantic search', + }; + + const quickActions: QuickAction[] = [ + { + title: 'Configure provider', + description: 'Set endpoint, API key, and worker toggle', + icon: , + href: '/embeddings/settings', + }, + { + title: 'New config', + description: 'Create a schema embedding configuration', + icon: , + href: '/embeddings/configs/new', + }, + { + title: 'Test search', + description: 'Run operator semantic search', + icon: , + href: '/embeddings/test', + }, + ]; + + const warnings = collectWarnings({ + status, + capabilitiesError: settledError(capabilitiesResult), + statusError: settledError(statusResult), + configsError: settledError(configsResult), + settingsError: settledError(settingsResult), + backfillsError: settledError(backfillsResult), + workersEnabled, + capabilitiesSupported: capabilities?.supported, + capabilitiesReason: capabilities?.reason, + }); + + return ( + } + moduleStatus={moduleStatus} + metrics={metricCards} + systemMetrics={settledValue(systemMetricsResult)} + quickActions={quickActions} + prometheusState={promAvailability?.state} + > +
+ {warnings.length > 0 ? ( +
+ {warnings.map((warning, index) => ( + + {warning.variant === 'destructive' ? ( + + ) : ( + + )} + {warning.title} + {warning.description} + + ))} +
+ ) : null} + +
+ + ); +} + +type OverviewWarning = { + title: string; + description: string; + variant: 'warning' | 'destructive'; +}; + +function collectWarnings(args: { + status?: EmbeddingsStatus; + capabilitiesError?: string; + statusError?: string; + configsError?: string; + settingsError?: string; + backfillsError?: string; + workersEnabled?: boolean; + capabilitiesSupported?: boolean; + capabilitiesReason?: string; +}): OverviewWarning[] { + const warnings: OverviewWarning[] = []; + + if (args.capabilitiesSupported === false) { + warnings.push({ + title: 'Vector capabilities unavailable', + description: + args.capabilitiesReason ?? + 'This database does not support vector storage and search.', + variant: 'destructive', + }); + } else if (args.capabilitiesError) { + warnings.push({ + title: 'Could not load capabilities', + description: args.capabilitiesError, + variant: 'warning', + }); + } + + if (args.workersEnabled === false) { + warnings.push({ + title: 'Workers disabled', + description: + 'Overview stays available. Enable workers in Settings to process jobs.', + variant: 'warning', + }); + } + + if (args.statusError) { + warnings.push({ + title: 'Status unavailable', + description: args.statusError, + variant: 'warning', + }); + } + + if (args.settingsError) { + warnings.push({ + title: 'Settings unavailable', + description: args.settingsError, + variant: 'warning', + }); + } + + if (args.configsError) { + warnings.push({ + title: 'Configs unavailable', + description: args.configsError, + variant: 'warning', + }); + } + + if (args.backfillsError) { + warnings.push({ + title: 'Backfill count unavailable', + description: args.backfillsError, + variant: 'warning', + }); + } + + for (const message of args.status?.warnings ?? []) { + warnings.push({ + title: 'Embeddings warning', + description: message, + variant: 'warning', + }); + } + + return warnings; +} diff --git a/src/app/(dashboard)/template.tsx b/src/app/(dashboard)/template.tsx index 206a304d3..3f1d5c825 100644 --- a/src/app/(dashboard)/template.tsx +++ b/src/app/(dashboard)/template.tsx @@ -36,6 +36,7 @@ const MODULE_NAMES: { [key: string]: string } = { authentication: 'Authentication', authorization: 'Authorization', database: 'Database', + embeddings: 'Embeddings', storage: 'Storage', chat: 'Chat', forms: 'Forms', @@ -52,11 +53,26 @@ const SEGMENT_LABELS: Record = { templates: 'Templates', logs: 'Logs & Devices', settings: 'Settings', - test: 'Test Send', }; -function formatBreadcrumbSegment(segment: string): string { +const MODULE_SEGMENT_LABELS: Record> = { + communications: { + test: 'Test Send', + }, + embeddings: { + configs: 'Configs', + backfills: 'Backfills', + test: 'Test Search', + new: 'New config', + }, +}; + +function formatBreadcrumbSegment(segment: string, moduleSlug?: string): string { + const moduleLabels = moduleSlug + ? MODULE_SEGMENT_LABELS[moduleSlug] + : undefined; return ( + moduleLabels?.[segment] ?? SEGMENT_LABELS[segment] ?? segment .split('-') @@ -208,7 +224,8 @@ export default function ModuleHeader({ {formatBreadcrumbSegment( - pathSegments[pathSegments.length - 1] + pathSegments[pathSegments.length - 1], + whichModule )} @@ -225,7 +242,10 @@ export default function ModuleHeader({ - {formatBreadcrumbSegment(pathSegments[1])} + {formatBreadcrumbSegment( + pathSegments[1], + whichModule + )} @@ -260,7 +280,8 @@ export default function ModuleHeader({ {formatBreadcrumbSegment( - pathSegments[pathSegments.length - 1] + pathSegments[pathSegments.length - 1], + whichModule )} diff --git a/src/components/dashboard/ActiveModuleGrid.tsx b/src/components/dashboard/ActiveModuleGrid.tsx index d27f4a631..b9a902832 100644 --- a/src/components/dashboard/ActiveModuleGrid.tsx +++ b/src/components/dashboard/ActiveModuleGrid.tsx @@ -5,6 +5,7 @@ import type { ResolvedModuleState } from '@/lib/prometheus/metrics'; import { Users, Database, + ScanSearch, HardDrive, MessageSquare, CreditCard, @@ -21,6 +22,7 @@ const MODULE_ICONS: Record = { authentication: Users, authorization: KeyRound, database: Database, + embeddings: ScanSearch, email: Mail, storage: HardDrive, functions: FunctionSquare, diff --git a/src/components/embeddings/EmbeddingsReadiness.tsx b/src/components/embeddings/EmbeddingsReadiness.tsx new file mode 100644 index 000000000..503f91986 --- /dev/null +++ b/src/components/embeddings/EmbeddingsReadiness.tsx @@ -0,0 +1,110 @@ +import Link from 'next/link'; +import { + AlertTriangle, + CheckCircle2, + CircleHelp, + Clock, + type LucideIcon, +} from 'lucide-react'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { cn } from '@/lib/utils'; +import { + getNextReadinessAction, + type ReadinessRow, + type ReadinessState, +} from '@/lib/models/embeddings'; + +const STATE_LABEL: Record = { + ready: 'Ready', + waiting: 'Waiting', + blocked: 'Blocked', + unknown: 'Unknown', +}; + +const STATE_ICON: Record = { + ready: CheckCircle2, + waiting: Clock, + blocked: AlertTriangle, + unknown: CircleHelp, +}; + +const STATE_ICON_CLASS: Record = { + ready: 'text-status-healthy', + waiting: 'text-status-warning', + blocked: 'text-status-critical', + unknown: 'text-status-unknown', +}; + +type EmbeddingsReadinessProps = { + rows: ReadinessRow[]; + className?: string; +}; + +export function EmbeddingsReadiness({ + rows, + className, +}: EmbeddingsReadinessProps) { + const next = getNextReadinessAction(rows); + + return ( + + + Readiness + {next ? ( +

+ Next: {next.detail} +

+ ) : ( +

+ Capabilities, provider, index, config, and workers are ready. +

+ )} + + +
    + {rows.map((row, index) => { + const Icon = STATE_ICON[row.state]; + return ( +
  • + +
    +

    + {row.label} +

    +

    + {row.detail} +

    +
    + + {STATE_LABEL[row.state]} + + {row.href && row.actionLabel ? ( + + {row.actionLabel} + + ) : null} +
    • + ); + })} +
+ + + ); +} diff --git a/src/components/navigation/navList.config.ts b/src/components/navigation/navList.config.ts index ff9d42677..3c6f7402d 100644 --- a/src/components/navigation/navList.config.ts +++ b/src/components/navigation/navList.config.ts @@ -8,6 +8,7 @@ import { LucideMail, MessagesSquare, Router, + ScanSearch, Settings, User, CreditCard, @@ -76,6 +77,17 @@ export const navGroups: NavGroup[] = [ { title: 'Settings', url: '/database/settings' }, ], }, + { + title: 'Embeddings', + url: '/embeddings', + icon: ScanSearch, + items: [ + { title: 'Configs', url: '/embeddings/configs' }, + { title: 'Backfills', url: '/embeddings/backfills' }, + { title: 'Test Search', url: '/embeddings/test' }, + { title: 'Settings', url: '/embeddings/settings' }, + ], + }, { title: 'Storage', url: '/storage', diff --git a/src/lib/models/embeddings/index.ts b/src/lib/models/embeddings/index.ts index d84532a3d..8f6b89bed 100644 --- a/src/lib/models/embeddings/index.ts +++ b/src/lib/models/embeddings/index.ts @@ -6,3 +6,4 @@ export * from './settings'; export * from './errors'; export * from './secrets'; export * from './normalize'; +export * from './readiness'; diff --git a/src/lib/models/embeddings/readiness.ts b/src/lib/models/embeddings/readiness.ts new file mode 100644 index 000000000..57bcc356f --- /dev/null +++ b/src/lib/models/embeddings/readiness.ts @@ -0,0 +1,292 @@ +import { + isVectorIndexQueryable, + VectorCapabilities, + VectorIndexDefinition, +} from '@/lib/models/embeddings/capabilities'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { isSecretConfigured } from '@/lib/models/embeddings/secrets'; +import { + EmbeddingsSettings, + OPENAI_COMPATIBLE_PROVIDER, +} from '@/lib/models/embeddings/settings'; + +export const READINESS_STATES = [ + 'ready', + 'waiting', + 'blocked', + 'unknown', +] as const; + +export type ReadinessState = (typeof READINESS_STATES)[number]; + +export const READINESS_ROW_IDS = [ + 'capabilities', + 'provider', + 'index', + 'config', + 'workers', +] as const; + +export type ReadinessRowId = (typeof READINESS_ROW_IDS)[number]; + +export type ReadinessRow = { + id: ReadinessRowId; + label: string; + state: ReadinessState; + detail: string; + href?: string; + actionLabel?: string; +}; + +export type SchemaIndexLookup = VectorIndexDefinition[] | 'unknown'; + +export type EmbeddingsReadinessInput = { + capabilities?: VectorCapabilities; + capabilitiesError?: string; + settings?: EmbeddingsSettings; + settingsError?: string; + configs?: EmbeddingConfig[]; + configsError?: string; + indexesBySchema?: Record; + selectedConfigId?: string; + workersEnabled?: boolean; +}; + +export function findMatchingIndex( + config: EmbeddingConfig, + indexes: VectorIndexDefinition[] +): VectorIndexDefinition | undefined { + return indexes.find( + index => + index.field === config.targetField && + index.dimensions === config.dimensions && + index.similarity === config.similarity + ); +} + +export function isProviderConfigured( + settings: EmbeddingsSettings | undefined +): boolean | undefined { + if (!settings) return undefined; + const name = settings.defaultProvider || OPENAI_COMPATIBLE_PROVIDER; + const provider = + settings.providers[name] ?? settings.providers[OPENAI_COMPATIBLE_PROVIDER]; + if (!provider) return false; + return ( + isSecretConfigured(provider.apiKey) && provider.endpoint.trim().length > 0 + ); +} + +export function isVectorStorageSearchReady( + capabilities: VectorCapabilities | undefined +): boolean | undefined { + if (!capabilities) return undefined; + return capabilities.supported && capabilities.storage && capabilities.search; +} + +function configHref(config?: EmbeddingConfig): string { + return config ? `/embeddings/configs/${config._id}` : '/embeddings/configs'; +} + +function evaluateIndexRow( + configs: EmbeddingConfig[] | undefined, + configsError: string | undefined, + indexesBySchema: Record | undefined, + selectedConfig?: EmbeddingConfig +): Pick { + if (configsError) { + return { state: 'unknown', detail: 'Config list unavailable' }; + } + if (!configs) { + return { state: 'unknown', detail: 'Index status unavailable' }; + } + + const scoped = selectedConfig ? [selectedConfig] : configs; + if (scoped.length === 0) { + return { + state: 'blocked', + detail: 'Create a config before an index can match', + href: '/embeddings/configs/new', + actionLabel: 'New config', + }; + } + + let sawUnknown = false; + let pending: EmbeddingConfig | undefined; + let failed: EmbeddingConfig | undefined; + let missing: EmbeddingConfig | undefined; + let readyConfig: EmbeddingConfig | undefined; + + for (const config of scoped) { + const lookup = indexesBySchema?.[config.schemaName]; + if (lookup == null || lookup === 'unknown') { + sawUnknown = true; + continue; + } + const match = findMatchingIndex(config, lookup); + if (!match) { + missing ??= config; + continue; + } + if (isVectorIndexQueryable(match)) { + readyConfig = config; + break; + } + if (match.status === 'failed') { + failed ??= config; + continue; + } + pending ??= config; + } + + if (readyConfig) { + return { state: 'ready', detail: 'Matching index is queryable' }; + } + if (failed) { + return { + state: 'blocked', + detail: 'Matching index failed', + href: configHref(selectedConfig ?? failed), + actionLabel: 'Open config', + }; + } + if (pending) { + return { + state: 'waiting', + detail: 'Matching index is not queryable yet', + href: configHref(selectedConfig ?? pending), + actionLabel: 'Open config', + }; + } + if (missing) { + return { + state: 'blocked', + detail: 'No index matches field, dimensions, and similarity', + href: configHref(selectedConfig ?? missing), + actionLabel: 'Open config', + }; + } + if (sawUnknown) { + return { state: 'unknown', detail: 'Index status unavailable' }; + } + return { + state: 'blocked', + detail: 'No index matches field, dimensions, and similarity', + href: configHref(selectedConfig), + actionLabel: selectedConfig ? 'Open config' : 'Open configs', + }; +} + +export function deriveEmbeddingsReadiness( + input: EmbeddingsReadinessInput +): ReadinessRow[] { + const selectedConfig = input.selectedConfigId + ? input.configs?.find(config => config._id === input.selectedConfigId) + : undefined; + + const capabilitiesReady = isVectorStorageSearchReady(input.capabilities); + const capabilitiesRow: ReadinessRow = { + id: 'capabilities', + label: 'Vector storage and search', + state: 'unknown', + detail: 'Capabilities unavailable', + }; + if (input.capabilitiesError) { + capabilitiesRow.detail = input.capabilitiesError; + } else if (capabilitiesReady === true) { + capabilitiesRow.state = 'ready'; + capabilitiesRow.detail = 'Storage and search are available'; + } else if (capabilitiesReady === false) { + capabilitiesRow.state = 'blocked'; + capabilitiesRow.detail = + input.capabilities?.reason ?? + 'This database does not support vector storage and search'; + capabilitiesRow.href = '/database'; + capabilitiesRow.actionLabel = 'Open Database'; + } + + const providerReady = isProviderConfigured(input.settings); + const providerRow: ReadinessRow = { + id: 'provider', + label: 'Provider configured', + state: 'unknown', + detail: 'Provider settings unavailable', + }; + if (input.settingsError) { + providerRow.detail = input.settingsError; + } else if (providerReady === true) { + providerRow.state = 'ready'; + providerRow.detail = 'Endpoint and API key are set'; + } else if (providerReady === false) { + providerRow.state = 'blocked'; + providerRow.detail = 'Set an endpoint and API key'; + providerRow.href = '/embeddings/settings'; + providerRow.actionLabel = 'Configure provider'; + } + + const indexEval = evaluateIndexRow( + input.configs, + input.configsError, + input.indexesBySchema, + selectedConfig + ); + const indexRow: ReadinessRow = { + id: 'index', + label: 'Matching index queryable', + ...indexEval, + }; + + const configRow: ReadinessRow = { + id: 'config', + label: 'Config enabled', + state: 'unknown', + detail: 'Config list unavailable', + }; + if (input.configsError) { + configRow.detail = input.configsError; + } else if (input.configs) { + const scoped = selectedConfig ? [selectedConfig] : input.configs; + if (scoped.length === 0) { + configRow.state = 'blocked'; + configRow.detail = 'Create a config to start embedding'; + configRow.href = '/embeddings/configs/new'; + configRow.actionLabel = 'New config'; + } else if (scoped.some(config => config.enabled)) { + configRow.state = 'ready'; + configRow.detail = selectedConfig + ? 'This config is enabled' + : 'At least one config is enabled'; + } else { + configRow.state = 'blocked'; + configRow.detail = selectedConfig + ? 'Enable this config after the index is ready' + : 'Enable a config after the index is ready'; + configRow.href = configHref(selectedConfig ?? scoped[0]); + configRow.actionLabel = selectedConfig ? 'Open config' : 'Open configs'; + } + } + + const workersRow: ReadinessRow = { + id: 'workers', + label: 'Workers enabled', + state: 'unknown', + detail: 'Worker status unavailable', + }; + if (input.workersEnabled === true) { + workersRow.state = 'ready'; + workersRow.detail = 'Module workers are running'; + } else if (input.workersEnabled === false) { + workersRow.state = 'blocked'; + workersRow.detail = 'Enable workers to process embeddings'; + workersRow.href = '/embeddings/settings'; + workersRow.actionLabel = 'Open settings'; + } + + return [capabilitiesRow, providerRow, indexRow, configRow, workersRow]; +} + +export function getNextReadinessAction( + rows: ReadinessRow[] +): ReadinessRow | undefined { + return rows.find(row => row.state !== 'ready' && row.href); +} diff --git a/src/lib/models/logs-viewer/constants.tsx b/src/lib/models/logs-viewer/constants.tsx index 6700a0a7a..949c70334 100644 --- a/src/lib/models/logs-viewer/constants.tsx +++ b/src/lib/models/logs-viewer/constants.tsx @@ -40,6 +40,7 @@ export const moduleNameByPath = { chat: 'chat', payments: 'payments', database: 'database', + embeddings: 'embeddings', router: 'router', settings: 'settings', communications: 'communications', diff --git a/src/lib/prometheus/metrics.ts b/src/lib/prometheus/metrics.ts index dc2999dda..25dc92401 100644 --- a/src/lib/prometheus/metrics.ts +++ b/src/lib/prometheus/metrics.ts @@ -661,6 +661,7 @@ const MODULE_CONFIGS = [ href: '/authorization', }, { name: 'database', displayName: 'Database', href: '/database' }, + { name: 'embeddings', displayName: 'Embeddings', href: '/embeddings' }, { name: 'email', displayName: 'Email', href: '/email' }, { name: 'storage', displayName: 'Storage', href: '/storage' }, { name: 'functions', displayName: 'Functions', href: '/functions' }, @@ -846,6 +847,7 @@ const CANONICAL_MODULES = [ 'authentication', 'authorization', 'database', + 'embeddings', 'email', 'storage', 'functions', diff --git a/src/lib/utils/module-utils.ts b/src/lib/utils/module-utils.ts index d4d3ce239..54c811503 100644 --- a/src/lib/utils/module-utils.ts +++ b/src/lib/utils/module-utils.ts @@ -54,6 +54,7 @@ export const MODULE_DISPLAY_NAMES: Record = { authentication: 'Authentication', authorization: 'Authorization', database: 'Database', + embeddings: 'Embeddings', storage: 'Storage', chat: 'Chat', email: 'Email', @@ -71,6 +72,7 @@ export const MODULE_URL_TO_NAME: Record = { '/authentication': 'authentication', '/authorization': 'authorization', '/database': 'database', + '/embeddings': 'embeddings', '/storage': 'storage', '/chat': 'chat', '/email': 'email', @@ -88,6 +90,7 @@ export const MODULE_NAME_TO_URL: Record = { authentication: 'authentication', authorization: 'authorization', database: 'database', + embeddings: 'embeddings', storage: 'storage', chat: 'chat', email: 'email', From ebcad93db839cd9e6c80fb17b1f517fdf4862822 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Tue, 8 Sep 2026 20:02:37 +0300 Subject: [PATCH 03/24] feat(embeddings): add config CRUD and index views Operators need list/create/detail flows that show matching index readiness, gate enablement, and confirm edits that can replace an index. --- .../embeddings/configs/[id]/loading.tsx | 21 ++ .../embeddings/configs/[id]/not-found.tsx | 34 +++ .../embeddings/configs/[id]/page.tsx | 99 +++++++++ .../(modules)/embeddings/configs/loading.tsx | 17 ++ .../embeddings/configs/new/loading.tsx | 16 ++ .../(modules)/embeddings/configs/new/page.tsx | 49 +++++ .../(modules)/embeddings/configs/page.tsx | 79 +++++++ .../(dashboard)/(modules)/embeddings/page.tsx | 38 +--- src/components/embeddings/configs/columns.tsx | 108 ++++++++++ .../embeddings/configs/config-detail.tsx | 79 +++++++ .../embeddings/configs/config-edit-form.tsx | 196 ++++++++++++++++++ .../embeddings/configs/config-form-fields.tsx | 126 +++++++++++ .../embeddings/configs/config-index-card.tsx | 175 ++++++++++++++++ .../embeddings/configs/configs-table.tsx | 32 +++ .../embeddings/configs/create-config-form.tsx | 111 ++++++++++ .../configs/delete-config-dialog.tsx | 50 +++++ .../configs/material-edit-dialog.tsx | 57 +++++ .../embeddings/configs/save-config-button.tsx | 57 +++++ src/components/embeddings/configs/schema.ts | 74 +++++++ .../configs/source-fields-picker.tsx | 87 ++++++++ .../embeddings/configs/use-save-shortcut.ts | 35 ++++ src/lib/api/embeddings/indexes.ts | 36 ++++ .../models/embeddings/config-change.test.ts | 72 +++++++ src/lib/models/embeddings/config-change.ts | 84 ++++++++ src/lib/models/embeddings/errors.ts | 5 + src/lib/models/embeddings/index-state.ts | 133 ++++++++++++ src/lib/models/embeddings/index.ts | 3 + .../models/embeddings/source-fields.test.ts | 89 ++++++++ src/lib/models/embeddings/source-fields.ts | 157 ++++++++++++++ 29 files changed, 2083 insertions(+), 36 deletions(-) create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/[id]/loading.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/[id]/not-found.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/[id]/page.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/loading.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/new/loading.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/new/page.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/configs/page.tsx create mode 100644 src/components/embeddings/configs/columns.tsx create mode 100644 src/components/embeddings/configs/config-detail.tsx create mode 100644 src/components/embeddings/configs/config-edit-form.tsx create mode 100644 src/components/embeddings/configs/config-form-fields.tsx create mode 100644 src/components/embeddings/configs/config-index-card.tsx create mode 100644 src/components/embeddings/configs/configs-table.tsx create mode 100644 src/components/embeddings/configs/create-config-form.tsx create mode 100644 src/components/embeddings/configs/delete-config-dialog.tsx create mode 100644 src/components/embeddings/configs/material-edit-dialog.tsx create mode 100644 src/components/embeddings/configs/save-config-button.tsx create mode 100644 src/components/embeddings/configs/schema.ts create mode 100644 src/components/embeddings/configs/source-fields-picker.tsx create mode 100644 src/components/embeddings/configs/use-save-shortcut.ts create mode 100644 src/lib/api/embeddings/indexes.ts create mode 100644 src/lib/models/embeddings/config-change.test.ts create mode 100644 src/lib/models/embeddings/config-change.ts create mode 100644 src/lib/models/embeddings/index-state.ts create mode 100644 src/lib/models/embeddings/source-fields.test.ts create mode 100644 src/lib/models/embeddings/source-fields.ts diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/[id]/loading.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/loading.tsx new file mode 100644 index 000000000..1c1447aa8 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/loading.tsx @@ -0,0 +1,21 @@ +import { Skeleton } from '@/components/ui/skeleton'; + +export default function EmbeddingConfigDetailLoading() { + return ( +
+
+
+ + +
+
+ + +
+
+ + + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/[id]/not-found.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/not-found.tsx new file mode 100644 index 000000000..4cbcba912 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/not-found.tsx @@ -0,0 +1,34 @@ +import { Button } from '@/components/ui/button'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { FileQuestion } from 'lucide-react'; +import Link from 'next/link'; + +export default function EmbeddingConfigNotFound() { + return ( +
+ + +
+ +
+ Config not found + + This embedding configuration does not exist or was deleted. Vector + fields and indexes on the schema are retained. + + + + + + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/[id]/page.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/page.tsx new file mode 100644 index 000000000..c4d719218 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/[id]/page.tsx @@ -0,0 +1,99 @@ +import { notFound } from 'next/navigation'; +import { ConfigDetail } from '@/components/embeddings/configs/config-detail'; +import { getSchemas } from '@/lib/api/database'; +import { + getEmbeddingConfig, + getEmbeddingsCapabilities, + getEmbeddingsSettings, + getEmbeddingsStatus, +} from '@/lib/api/embeddings'; +import { resolveIndexesBySchema } from '@/lib/api/embeddings/indexes'; +import { + deriveEmbeddingsReadiness, + EmbeddingConfig, + formatEmbeddingsApiError, + isEmbeddingsNotFound, + listEligibleSchemas, + toEmbeddingSchemaChoice, +} from '@/lib/models/embeddings'; + +function settledValue(result: PromiseSettledResult): T | undefined { + return result.status === 'fulfilled' ? result.value : undefined; +} + +function settledError( + result: PromiseSettledResult +): string | undefined { + if (result.status !== 'rejected') return undefined; + return formatEmbeddingsApiError(result.reason); +} + +export default async function EmbeddingConfigDetailPage(props: { + params: Promise<{ id: string }>; +}) { + const { id } = await props.params; + + let config: EmbeddingConfig; + try { + config = await getEmbeddingConfig(id); + } catch (error) { + if (isEmbeddingsNotFound(error)) notFound(); + throw error; + } + + const [capabilitiesResult, settingsResult, statusResult, schemasResult] = + await Promise.allSettled([ + getEmbeddingsCapabilities(config.schemaName), + getEmbeddingsSettings(), + getEmbeddingsStatus(config.schemaName), + getSchemas({ limit: 1000, enabled: true }), + ]); + + const indexesBySchema = await resolveIndexesBySchema([config.schemaName]); + const lookup = indexesBySchema[config.schemaName]; + const status = settledValue(statusResult); + const capabilities = + settledValue(capabilitiesResult)?.capabilities ?? status?.capabilities; + const settings = settledValue(settingsResult)?.config; + const rawSchemas = settledValue(schemasResult)?.schemas ?? []; + const eligibleSchemas = listEligibleSchemas(rawSchemas); + const currentSchema = rawSchemas.find( + schema => schema.name === config.schemaName + ); + const schemaChoices = currentSchema + ? [ + toEmbeddingSchemaChoice(currentSchema), + ...eligibleSchemas.filter(schema => schema.name !== currentSchema.name), + ] + : [ + toEmbeddingSchemaChoice({ + name: config.schemaName, + ownerModule: 'database', + fields: Object.fromEntries( + config.sourceFields.map(name => [name, { type: 'String' }]) + ), + }), + ...eligibleSchemas, + ]; + + const rows = deriveEmbeddingsReadiness({ + capabilities, + capabilitiesError: settledError(capabilitiesResult), + settings, + settingsError: settledError(settingsResult), + configs: [config], + indexesBySchema, + selectedConfigId: config._id, + workersEnabled: status?.enabled ?? settings?.enabled, + }); + + return ( + + ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/loading.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/loading.tsx new file mode 100644 index 000000000..8a1b144bc --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/loading.tsx @@ -0,0 +1,17 @@ +import { Skeleton } from '@/components/ui/skeleton'; + +export default function EmbeddingConfigsLoading() { + return ( +
+
+
+ + +
+ +
+ + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/new/loading.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/new/loading.tsx new file mode 100644 index 000000000..ee5f0c1df --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/new/loading.tsx @@ -0,0 +1,16 @@ +import { Skeleton } from '@/components/ui/skeleton'; + +export default function NewEmbeddingConfigLoading() { + return ( +
+
+ + +
+ + + + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/new/page.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/new/page.tsx new file mode 100644 index 000000000..1fd2d93f8 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/new/page.tsx @@ -0,0 +1,49 @@ +import { CreateConfigForm } from '@/components/embeddings/configs/create-config-form'; +import { + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { getSchemas } from '@/lib/api/database'; +import { getEmbeddingsSettings } from '@/lib/api/embeddings'; +import { + listEligibleSchemas, + OPENAI_COMPATIBLE_PROVIDER, +} from '@/lib/models/embeddings'; + +function settledValue(result: PromiseSettledResult): T | undefined { + return result.status === 'fulfilled' ? result.value : undefined; +} + +export default async function NewEmbeddingConfigPage() { + const [schemasResult, settingsResult] = await Promise.allSettled([ + getSchemas({ limit: 1000, enabled: true }), + getEmbeddingsSettings(), + ]); + + const schemas = listEligibleSchemas( + settledValue(schemasResult)?.schemas ?? [] + ); + const settings = settledValue(settingsResult)?.config; + const provider = settings?.defaultProvider || OPENAI_COMPATIBLE_PROVIDER; + const model = settings?.providers[provider]?.model ?? ''; + + return ( +
+ +
+ New config + + Choose a schema and eligible string fields. The config stays + disabled until a matching index is queryable. + +
+ + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/configs/page.tsx b/src/app/(dashboard)/(modules)/embeddings/configs/page.tsx new file mode 100644 index 000000000..af64148c3 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/configs/page.tsx @@ -0,0 +1,79 @@ +import Link from 'next/link'; +import { Plus } from 'lucide-react'; +import { ConfigsTable } from '@/components/embeddings/configs/configs-table'; +import { ErrorCard } from '@/components/error/ErrorCard'; +import { Button } from '@/components/ui/button'; +import { + PageActions, + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { getBackfills, getEmbeddingConfigs } from '@/lib/api/embeddings'; +import { resolveIndexesBySchema } from '@/lib/api/embeddings/indexes'; +import { + buildConfigListRows, + formatEmbeddingsApiError, +} from '@/lib/models/embeddings'; + +function settledValue(result: PromiseSettledResult): T | undefined { + return result.status === 'fulfilled' ? result.value : undefined; +} + +function settledError( + result: PromiseSettledResult +): string | undefined { + if (result.status !== 'rejected') return undefined; + return formatEmbeddingsApiError(result.reason); +} + +export default async function EmbeddingConfigsPage() { + const [configsResult, backfillsResult] = await Promise.allSettled([ + getEmbeddingConfigs(), + getBackfills({ skip: 0, limit: 100 }), + ]); + + const configsError = settledError(configsResult); + if (configsError) { + return ( + + Embeddings overview + + } + /> + ); + } + + const configs = settledValue(configsResult) ?? []; + const runs = settledValue(backfillsResult)?.runs ?? []; + const indexesBySchema = await resolveIndexesBySchema( + configs.map(config => config.schemaName) + ); + const rows = buildConfigListRows(configs, indexesBySchema, runs); + + return ( +
+ +
+ Configs + + Schema embedding configurations and matching index readiness. + +
+ + + + + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/page.tsx b/src/app/(dashboard)/(modules)/embeddings/page.tsx index 2b0bd1ad2..048a57fe8 100644 --- a/src/app/(dashboard)/(modules)/embeddings/page.tsx +++ b/src/app/(dashboard)/(modules)/embeddings/page.tsx @@ -19,14 +19,12 @@ import { getEmbeddingsSettings, getEmbeddingsStatus, } from '@/lib/api/embeddings'; -import { getSchemas, getSchemaVectorIndexes } from '@/lib/api/database'; +import { resolveIndexesBySchema } from '@/lib/api/embeddings/indexes'; import { deriveEmbeddingsReadiness, - EmbeddingConfig, EmbeddingsQueueCounts, EmbeddingsStatus, formatEmbeddingsApiError, - SchemaIndexLookup, } from '@/lib/models/embeddings'; import { getPrometheusAvailability } from '@/lib/observability/prometheusAvailability'; import { @@ -72,38 +70,6 @@ function addQueues( }; } -async function resolveIndexesBySchema( - configs: EmbeddingConfig[] -): Promise> { - const schemaNames = [...new Set(configs.map(config => config.schemaName))]; - if (schemaNames.length === 0) return {}; - - let schemas: { name: string; _id: string }[] | undefined; - try { - const response = await getSchemas({ limit: 1000 }); - schemas = response.schemas; - } catch { - return Object.fromEntries(schemaNames.map(name => [name, 'unknown'])); - } - - const idByName = new Map(schemas.map(schema => [schema.name, schema._id])); - const lookups = await Promise.allSettled( - schemaNames.map(async (name): Promise<[string, SchemaIndexLookup]> => { - const schemaId = idByName.get(name); - if (!schemaId) return [name, []]; - const { indexes } = await getSchemaVectorIndexes(schemaId); - return [name, indexes]; - }) - ); - - const map: Record = {}; - for (const [index, name] of schemaNames.entries()) { - const result = lookups[index]; - map[name] = result.status === 'fulfilled' ? result.value[1] : 'unknown'; - } - return map; -} - function metricValue( value: number | undefined, unavailable: boolean @@ -151,7 +117,7 @@ export default async function EmbeddingsDashboard() { const promAvailability = settledValue(promAvailabilityResult); const indexesBySchema = configs - ? await resolveIndexesBySchema(configs) + ? await resolveIndexesBySchema(configs.map(config => config.schemaName)) : undefined; const workersEnabled = status?.enabled ?? settings?.enabled; diff --git a/src/components/embeddings/configs/columns.tsx b/src/components/embeddings/configs/columns.tsx new file mode 100644 index 000000000..0d12fbe2c --- /dev/null +++ b/src/components/embeddings/configs/columns.tsx @@ -0,0 +1,108 @@ +'use client'; + +import { ColumnDef } from '@tanstack/react-table'; +import Link from 'next/link'; +import { Badge } from '@/components/ui/badge'; +import { + configIndexStateLabel, + EmbeddingConfigListRow, + similarityLabel, +} from '@/lib/models/embeddings/index-state'; +import { cn } from '@/lib/utils'; + +const INDEX_STATE_CLASS: Record = + { + ready: 'text-status-healthy', + pending: 'text-status-warning', + failed: 'text-status-critical', + missing: 'text-status-critical', + unknown: 'text-status-unknown', + }; + +export function getConfigColumns(): ColumnDef[] { + return [ + { + id: 'schema', + header: 'Schema', + cell: ({ row }) => ( +
+ + + {row.original.config.schemaName} + + +
+ ), + }, + { + id: 'target', + header: 'Target', + cell: ({ row }) => ( +
{row.original.config.targetField}
+ ), + }, + { + id: 'provider', + header: 'Provider / model', + cell: ({ row }) => ( +
+ {row.original.config.provider}/{row.original.config.model} +
+ ), + }, + { + id: 'dimensions', + header: 'Dimensions', + cell: ({ row }) => ( + {row.original.config.dimensions} + ), + }, + { + id: 'similarity', + header: 'Similarity', + cell: ({ row }) => similarityLabel(row.original.config.similarity), + }, + { + id: 'enabled', + header: 'Enabled', + cell: ({ row }) => ( + + {row.original.config.enabled ? 'Enabled' : 'Disabled'} + + ), + }, + { + id: 'backfill', + header: 'Latest backfill', + cell: ({ row }) => { + const run = row.original.latestBackfill; + if (!run) return —; + return ( + + {run.state.charAt(0).toUpperCase() + run.state.slice(1)} + + ); + }, + }, + { + id: 'index', + header: 'Index', + cell: ({ row }) => ( + + {configIndexStateLabel(row.original.indexState)} + + ), + }, + ]; +} diff --git a/src/components/embeddings/configs/config-detail.tsx b/src/components/embeddings/configs/config-detail.tsx new file mode 100644 index 000000000..5acb02b57 --- /dev/null +++ b/src/components/embeddings/configs/config-detail.tsx @@ -0,0 +1,79 @@ +'use client'; + +import Link from 'next/link'; +import { EmbeddingsReadiness } from '@/components/embeddings/EmbeddingsReadiness'; +import { ConfigEditForm } from '@/components/embeddings/configs/config-edit-form'; +import { ConfigIndexCard } from '@/components/embeddings/configs/config-index-card'; +import { + PageActions, + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { Button } from '@/components/ui/button'; +import { VectorCapabilities } from '@/lib/models/embeddings/capabilities'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { canEnableEmbeddingConfig } from '@/lib/models/embeddings/index-state'; +import { + findMatchingIndex, + ReadinessRow, + SchemaIndexLookup, +} from '@/lib/models/embeddings/readiness'; +import { EmbeddingSchemaChoice } from '@/lib/models/embeddings/source-fields'; + +type ConfigDetailProps = { + config: EmbeddingConfig; + schemas: EmbeddingSchemaChoice[]; + lookup?: SchemaIndexLookup; + capabilities?: VectorCapabilities; + readinessRows: ReadinessRow[]; +}; + +export function ConfigDetail({ + config, + schemas, + lookup, + capabilities, + readinessRows, +}: ConfigDetailProps) { + const matchingIndex = + lookup && lookup !== 'unknown' + ? findMatchingIndex(config, lookup) + : undefined; + const enableAllowed = canEnableEmbeddingConfig({ + capabilities, + matchingIndex, + }); + + return ( +
+ +
+ {config.schemaName} + + Target field {config.targetField}. Indexes stay read-only here. + +
+ + + + + + + + +
+ ); +} diff --git a/src/components/embeddings/configs/config-edit-form.tsx b/src/components/embeddings/configs/config-edit-form.tsx new file mode 100644 index 000000000..a9ab38ae4 --- /dev/null +++ b/src/components/embeddings/configs/config-edit-form.tsx @@ -0,0 +1,196 @@ +'use client'; + +import { useCallback, useState } from 'react'; +import { useForm } from 'react-hook-form'; +import { useRouter } from 'next/navigation'; +import { Form } from '@/components/ui/form'; +import { Button } from '@/components/ui/button'; +import { ConfigFormFields } from '@/components/embeddings/configs/config-form-fields'; +import { DeleteConfigDialog } from '@/components/embeddings/configs/delete-config-dialog'; +import { MaterialEditDialog } from '@/components/embeddings/configs/material-edit-dialog'; +import { SaveConfigButton } from '@/components/embeddings/configs/save-config-button'; +import { + embeddingConfigFormSchema, + EmbeddingConfigFormValues, +} from '@/components/embeddings/configs/schema'; +import { + useSaveShortcut, + useSaveShortcutLabel, +} from '@/components/embeddings/configs/use-save-shortcut'; +import { + deleteEmbeddingConfig, + upsertEmbeddingConfig, +} from '@/lib/api/embeddings'; +import { toast } from '@/lib/hooks/use-toast'; +import { + diffMaterialEmbeddingConfig, + isInPlaceDimensionChange, + requiresIndexRecreation, + toMaterialEmbeddingConfig, +} from '@/lib/models/embeddings/config-change'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { formatEmbeddingsApiError } from '@/lib/models/embeddings/errors'; +import { EmbeddingSchemaChoice } from '@/lib/models/embeddings/source-fields'; +import { rhfZodResolver } from '@/lib/zod-form'; + +type ConfigEditFormProps = { + config: EmbeddingConfig; + schemas: EmbeddingSchemaChoice[]; + enableAllowed: boolean; +}; + +export function ConfigEditForm({ + config, + schemas, + enableAllowed, +}: ConfigEditFormProps) { + const router = useRouter(); + const shortcutLabel = useSaveShortcutLabel(); + const existing = toMaterialEmbeddingConfig(config); + const [materialOpen, setMaterialOpen] = useState(false); + const [deleteOpen, setDeleteOpen] = useState(false); + const [deleting, setDeleting] = useState(false); + const [pendingValues, setPendingValues] = + useState(null); + + const form = useForm({ + resolver: rhfZodResolver(embeddingConfigFormSchema), + mode: 'onChange', + defaultValues: { + schemaName: config.schemaName, + sourceFields: config.sourceFields, + targetField: config.targetField, + provider: config.provider, + model: config.model, + dimensions: config.dimensions, + similarity: config.similarity, + enabled: config.enabled, + }, + }); + + const persist = useCallback( + async (values: EmbeddingConfigFormValues) => { + try { + const result = await upsertEmbeddingConfig({ + ...values, + enabled: values.enabled, + }); + toast({ + title: 'Config saved', + description: + result.warnings.length > 0 ? result.warnings.join(' ') : undefined, + }); + setMaterialOpen(false); + setPendingValues(null); + if (result.config._id !== config._id) { + router.push(`/embeddings/configs/${result.config._id}`); + return; + } + router.refresh(); + } catch (error) { + toast({ + title: 'Could not save config', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + } + }, + [config._id, router] + ); + + const submit = useCallback( + (values: EmbeddingConfigFormValues) => { + const next = toMaterialEmbeddingConfig(values); + if (isInPlaceDimensionChange(existing, next)) { + form.setError('dimensions', { + type: 'manual', + message: + 'Changing dimensions on the same target field is not allowed. Choose a new target field first.', + }); + return; + } + const changed = diffMaterialEmbeddingConfig(existing, next); + if (changed.length > 0) { + setPendingValues(values); + setMaterialOpen(true); + return; + } + void persist(values); + }, + [existing, form, persist] + ); + + const dirty = form.formState.isDirty; + const submitting = form.formState.isSubmitting; + useSaveShortcut( + dirty && !submitting && !materialOpen && !deleteOpen, + useCallback(() => { + void form.handleSubmit(submit)(); + }, [form, submit]) + ); + + const pendingMaterial = pendingValues + ? toMaterialEmbeddingConfig(pendingValues) + : existing; + const changed = diffMaterialEmbeddingConfig(existing, pendingMaterial); + + return ( + <> +
+ + +
+ + +
+ + + { + if (pendingValues) void persist(pendingValues); + }} + /> + { + setDeleting(true); + deleteEmbeddingConfig(config._id) + .then(() => { + toast({ title: 'Config deleted' }); + router.push('/embeddings/configs'); + }) + .catch(error => { + setDeleting(false); + toast({ + title: 'Could not delete config', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + }); + }} + /> + + ); +} diff --git a/src/components/embeddings/configs/config-form-fields.tsx b/src/components/embeddings/configs/config-form-fields.tsx new file mode 100644 index 000000000..cd3d8b734 --- /dev/null +++ b/src/components/embeddings/configs/config-form-fields.tsx @@ -0,0 +1,126 @@ +'use client'; + +import { useMemo } from 'react'; +import { useFormContext, useWatch } from 'react-hook-form'; +import { InputField } from '@/components/ui/form-inputs/InputField'; +import SelectField from '@/components/ui/form-inputs/SelectField'; +import SwitchField from '@/components/ui/form-inputs/SwitchField'; +import { SourceFieldsPicker } from '@/components/embeddings/configs/source-fields-picker'; +import { VECTOR_SIMILARITIES } from '@/lib/models/embeddings/config'; +import { similarityLabel } from '@/lib/models/embeddings/index-state'; +import { + EmbeddingSchemaChoice, + listSourceFieldChoices, +} from '@/lib/models/embeddings/source-fields'; + +type ConfigFormFieldsProps = { + schemas: EmbeddingSchemaChoice[]; + schemaLocked?: boolean; + enableAllowed: boolean; + enableBlockedReason?: string; +}; + +export function ConfigFormFields({ + schemas, + schemaLocked = false, + enableAllowed, + enableBlockedReason, +}: ConfigFormFieldsProps) { + const { control, setValue } = useFormContext(); + const schemaNameRaw = useWatch({ control, name: 'schemaName' }); + const sourceFieldsRaw = useWatch({ control, name: 'sourceFields' }); + const enabledRaw = useWatch({ control, name: 'enabled' }); + const schemaName = typeof schemaNameRaw === 'string' ? schemaNameRaw : ''; + const sourceFields = Array.isArray(sourceFieldsRaw) + ? sourceFieldsRaw.filter((item): item is string => typeof item === 'string') + : []; + const enabled = enabledRaw === true; + + const selectedSchema = schemas.find(schema => schema.name === schemaName); + const choices = useMemo( + () => listSourceFieldChoices(selectedSchema?.fields ?? {}, sourceFields), + [selectedSchema, sourceFields] + ); + + return ( +
+ schema.name)} + onValueChange={ + schemaLocked + ? undefined + : value => { + setValue('schemaName', value, { + shouldDirty: true, + shouldValidate: true, + }); + setValue('sourceFields', [], { + shouldDirty: true, + shouldValidate: true, + }); + } + } + description={ + schemaLocked + ? 'Schema cannot change after create.' + : schemas.length === 0 + ? 'No eligible Database schemas are available.' + : 'Hidden, system, and embeddings-owned schemas are omitted.' + } + /> + +
+ +
+ + + + ({ + value, + label: similarityLabel(value), + }))} + /> +
+ +

+ {enableAllowed + ? 'Documents will embed after the matching index is queryable.' + : (enableBlockedReason ?? + 'Enablement stays off until capabilities and a matching index are ready.')} +

+
+
+ ); +} diff --git a/src/components/embeddings/configs/config-index-card.tsx b/src/components/embeddings/configs/config-index-card.tsx new file mode 100644 index 000000000..759a354c3 --- /dev/null +++ b/src/components/embeddings/configs/config-index-card.tsx @@ -0,0 +1,175 @@ +'use client'; + +import Link from 'next/link'; +import { + AlertTriangle, + CheckCircle2, + CircleHelp, + Clock, + type LucideIcon, +} from 'lucide-react'; +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { + isVectorIndexQueryable, + VectorIndexDefinition, +} from '@/lib/models/embeddings/capabilities'; +import { + configIndexStateLabel, + parseIndexGeneration, + resolveConfigIndexState, + similarityLabel, +} from '@/lib/models/embeddings/index-state'; +import { + findMatchingIndex, + SchemaIndexLookup, +} from '@/lib/models/embeddings/readiness'; +import { cn } from '@/lib/utils'; + +const STATE_ICON: Record< + ReturnType, + LucideIcon +> = { + ready: CheckCircle2, + pending: Clock, + failed: AlertTriangle, + missing: AlertTriangle, + unknown: CircleHelp, +}; + +const STATE_CLASS: Record< + ReturnType, + string +> = { + ready: 'text-status-healthy', + pending: 'text-status-warning', + failed: 'text-status-critical', + missing: 'text-status-critical', + unknown: 'text-status-unknown', +}; + +type ConfigIndexCardProps = { + config: EmbeddingConfig; + lookup?: SchemaIndexLookup; +}; + +export function ConfigIndexCard({ config, lookup }: ConfigIndexCardProps) { + const match = + lookup && lookup !== 'unknown' + ? findMatchingIndex(config, lookup) + : undefined; + const state = resolveConfigIndexState(config, lookup); + const Icon = STATE_ICON[state]; + const parsed = parseIndexGeneration(match?.name); + const warning = indexWarning(state, match); + + return ( + + + Matching index +

+ Embeddings provisions this index. It is read-only here. +

+ + +
+ + + {configIndexStateLabel(state)} + + {match ? ( + + {isVectorIndexQueryable(match) ? 'Queryable' : 'Not queryable'} + + ) : null} +
+
+ + 0 ? String(parsed.generation) : '—'} + /> + + + + +
+ {warning ? ( + + + {warning.title} + + {warning.description}{' '} + {warning.href ? ( + + {warning.action} + + ) : null} + + + ) : null} + + + ); +} + +function IndexFact({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function indexWarning( + state: ReturnType, + match?: VectorIndexDefinition +): + | { title: string; description: string; href?: string; action?: string } + | undefined { + switch (state) { + case 'ready': + return undefined; + case 'pending': + return { + title: 'Index is not queryable yet', + description: + 'Keep this config disabled until the matching index becomes ready.', + }; + case 'failed': + return { + title: 'Matching index failed', + description: match?.name + ? `${match.name} failed. Inspect Database indexes for this schema.` + : 'The matching index failed. Inspect Database indexes for this schema.', + href: '/database/models', + action: 'Open Database', + }; + case 'missing': + return { + title: 'No matching index', + description: + 'No index matches this field, dimensions, and similarity yet. Save the config to provision one.', + }; + case 'unknown': + return { + title: 'Index status unavailable', + description: 'Index lookup failed. Refresh this page and try again.', + }; + default: { + const exhaustive: never = state; + return exhaustive; + } + } +} diff --git a/src/components/embeddings/configs/configs-table.tsx b/src/components/embeddings/configs/configs-table.tsx new file mode 100644 index 000000000..235618e6a --- /dev/null +++ b/src/components/embeddings/configs/configs-table.tsx @@ -0,0 +1,32 @@ +'use client'; + +import Link from 'next/link'; +import { Plus } from 'lucide-react'; +import { DataTable } from '@/components/ui/data-table'; +import { Button } from '@/components/ui/button'; +import { getConfigColumns } from '@/components/embeddings/configs/columns'; +import { EmbeddingConfigListRow } from '@/lib/models/embeddings/index-state'; + +type ConfigsTableProps = { + rows: EmbeddingConfigListRow[]; +}; + +export function ConfigsTable({ rows }: ConfigsTableProps) { + return ( + +
+

No embedding configs

+

+ Create a config to provision a matching vector index. Keep it disabled + until that index is queryable, then run a backfill. +

+ +
+ + ); +} diff --git a/src/components/embeddings/configs/create-config-form.tsx b/src/components/embeddings/configs/create-config-form.tsx new file mode 100644 index 000000000..ef349ec10 --- /dev/null +++ b/src/components/embeddings/configs/create-config-form.tsx @@ -0,0 +1,111 @@ +'use client'; + +import { useCallback } from 'react'; +import { useForm } from 'react-hook-form'; +import { useRouter } from 'next/navigation'; +import { Form } from '@/components/ui/form'; +import { Button } from '@/components/ui/button'; +import { ConfigFormFields } from '@/components/embeddings/configs/config-form-fields'; +import { SaveConfigButton } from '@/components/embeddings/configs/save-config-button'; +import { + defaultConfigFormValues, + embeddingConfigFormSchema, + EmbeddingConfigFormValues, +} from '@/components/embeddings/configs/schema'; +import { + useSaveShortcut, + useSaveShortcutLabel, +} from '@/components/embeddings/configs/use-save-shortcut'; +import { upsertEmbeddingConfig } from '@/lib/api/embeddings'; +import { toast } from '@/lib/hooks/use-toast'; +import { formatEmbeddingsApiError } from '@/lib/models/embeddings/errors'; +import { EmbeddingSchemaChoice } from '@/lib/models/embeddings/source-fields'; +import { rhfZodResolver } from '@/lib/zod-form'; + +type CreateConfigFormProps = { + schemas: EmbeddingSchemaChoice[]; + defaultProvider: string; + defaultModel: string; +}; + +export function CreateConfigForm({ + schemas, + defaultProvider, + defaultModel, +}: CreateConfigFormProps) { + const router = useRouter(); + const shortcutLabel = useSaveShortcutLabel(); + const form = useForm({ + resolver: rhfZodResolver(embeddingConfigFormSchema), + mode: 'onChange', + defaultValues: defaultConfigFormValues({ + provider: defaultProvider, + model: defaultModel, + }), + }); + + const submit = useCallback( + async (values: EmbeddingConfigFormValues) => { + try { + const result = await upsertEmbeddingConfig({ + ...values, + enabled: false, + }); + toast({ + title: 'Config created', + description: + result.warnings.length > 0 + ? result.warnings.join(' ') + : 'The config is disabled until the matching index is queryable.', + }); + router.push(`/embeddings/configs/${result.config._id}`); + } catch (error) { + toast({ + title: 'Could not create config', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + } + }, + [router] + ); + + const dirty = form.formState.isDirty; + const submitting = form.formState.isSubmitting; + useSaveShortcut( + dirty && !submitting, + useCallback(() => { + void form.handleSubmit(submit)(); + }, [form, submit]) + ); + + return ( +
+ + +
+ + +
+ + + ); +} diff --git a/src/components/embeddings/configs/delete-config-dialog.tsx b/src/components/embeddings/configs/delete-config-dialog.tsx new file mode 100644 index 000000000..cda3a1431 --- /dev/null +++ b/src/components/embeddings/configs/delete-config-dialog.tsx @@ -0,0 +1,50 @@ +'use client'; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; + +type DeleteConfigDialogProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; + pending?: boolean; +}; + +export function DeleteConfigDialog({ + open, + onOpenChange, + onConfirm, + pending = false, +}: DeleteConfigDialogProps) { + return ( + + + + Delete this config? + + This deletes the embedding configuration. Vector fields and indexes + on the schema are retained. + + + + Cancel + + {pending ? 'Deleting…' : 'Delete config'} + + + + + ); +} diff --git a/src/components/embeddings/configs/material-edit-dialog.tsx b/src/components/embeddings/configs/material-edit-dialog.tsx new file mode 100644 index 000000000..f8ce8854a --- /dev/null +++ b/src/components/embeddings/configs/material-edit-dialog.tsx @@ -0,0 +1,57 @@ +'use client'; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; + +type MaterialEditDialogProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; + recreatesIndex: boolean; + createsNewConfig: boolean; + pending?: boolean; +}; + +export function MaterialEditDialog({ + open, + onOpenChange, + onConfirm, + recreatesIndex, + createsNewConfig, + pending = false, +}: MaterialEditDialogProps) { + return ( + + + + Save material changes? + + Saving these changes can replace the matching vector index and + requires a backfill. Existing vectors stay stale until that backfill + completes. + {recreatesIndex + ? ' Wait until the index is queryable before searching or backfilling.' + : null} + {createsNewConfig + ? ' Changing the target field creates a new configuration. The previous target field config is kept.' + : null} + + + + Cancel + + {pending ? 'Saving…' : 'Save changes'} + + + + + ); +} diff --git a/src/components/embeddings/configs/save-config-button.tsx b/src/components/embeddings/configs/save-config-button.tsx new file mode 100644 index 000000000..39d7277ab --- /dev/null +++ b/src/components/embeddings/configs/save-config-button.tsx @@ -0,0 +1,57 @@ +'use client'; + +import { Loader2, Save } from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; + +type SaveConfigButtonProps = { + dirty: boolean; + submitting: boolean; + label: string; + shortcutLabel: string; +}; + +export function SaveConfigButton({ + dirty, + submitting, + label, + shortcutLabel, +}: SaveConfigButtonProps) { + return ( + + + + + + +

+ {dirty + ? `Save configuration ${shortcutLabel}` + : 'No unsaved changes'} +

+ + + + ); +} diff --git a/src/components/embeddings/configs/schema.ts b/src/components/embeddings/configs/schema.ts new file mode 100644 index 000000000..617b91cfd --- /dev/null +++ b/src/components/embeddings/configs/schema.ts @@ -0,0 +1,74 @@ +import { z } from 'zod'; +import { VECTOR_SIMILARITIES } from '@/lib/models/embeddings/config'; +import { + isInPlaceDimensionChange, + MaterialEmbeddingConfig, +} from '@/lib/models/embeddings/config-change'; +import { isValidSchemaOrTargetName } from '@/lib/models/embeddings/source-fields'; + +const NAME_MESSAGE = + 'Use a letter or underscore first, then letters, numbers, or underscores'; + +export const embeddingConfigFormSchema = z.object({ + schemaName: z + .string() + .min(1, 'Schema is required') + .refine(isValidSchemaOrTargetName, NAME_MESSAGE), + sourceFields: z + .array(z.string().min(1)) + .min(1, 'Select at least one source field'), + targetField: z + .string() + .min(1, 'Target field is required') + .refine(isValidSchemaOrTargetName, NAME_MESSAGE), + provider: z.string().min(1, 'Provider is required'), + model: z.string().min(1, 'Model is required'), + dimensions: z.coerce + .number({ error: 'Dimensions must be a positive integer' }) + .int('Dimensions must be a positive integer') + .positive('Dimensions must be a positive integer'), + similarity: z.enum(VECTOR_SIMILARITIES), + enabled: z.boolean(), +}); + +export type EmbeddingConfigFormValues = z.infer< + typeof embeddingConfigFormSchema +>; + +export function defaultConfigFormValues(args: { + provider?: string; + model?: string; +}): EmbeddingConfigFormValues { + return { + schemaName: '', + sourceFields: [], + targetField: '', + provider: args.provider || 'openai-compatible', + model: args.model || '', + dimensions: 1536, + similarity: 'cosine', + enabled: false, + }; +} + +export function embeddingConfigEditSchema(existing: MaterialEmbeddingConfig) { + return embeddingConfigFormSchema.superRefine((value, ctx) => { + if ( + isInPlaceDimensionChange(existing, { + provider: value.provider, + model: value.model, + dimensions: value.dimensions, + sourceFields: value.sourceFields, + targetField: value.targetField, + similarity: value.similarity, + }) + ) { + ctx.addIssue({ + code: 'custom', + path: ['dimensions'], + message: + 'Changing dimensions on the same target field is not allowed. Choose a new target field first.', + }); + } + }); +} diff --git a/src/components/embeddings/configs/source-fields-picker.tsx b/src/components/embeddings/configs/source-fields-picker.tsx new file mode 100644 index 000000000..ddd8e0d33 --- /dev/null +++ b/src/components/embeddings/configs/source-fields-picker.tsx @@ -0,0 +1,87 @@ +'use client'; + +import { useFormContext } from 'react-hook-form'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { SourceFieldChoice } from '@/lib/models/embeddings/source-fields'; +import { cn } from '@/lib/utils'; + +type SourceFieldsPickerProps = { + choices: SourceFieldChoice[]; +}; + +export function SourceFieldsPicker({ choices }: SourceFieldsPickerProps) { + const { control } = useFormContext(); + + return ( + { + const selected = Array.isArray(field.value) + ? field.value.filter( + (item): item is string => typeof item === 'string' + ) + : []; + return ( + + Source fields + + String fields only. Hidden and sensitive fields are not offered. + + {choices.length === 0 ? ( +

+ No eligible string fields on this schema. +

+ ) : ( +
+ {choices.map(choice => { + const checked = selected.includes(choice.name); + return ( + + ); + })} +
+ )} + + + ); + }} + /> + ); +} diff --git a/src/components/embeddings/configs/use-save-shortcut.ts b/src/components/embeddings/configs/use-save-shortcut.ts new file mode 100644 index 000000000..0d5123e34 --- /dev/null +++ b/src/components/embeddings/configs/use-save-shortcut.ts @@ -0,0 +1,35 @@ +'use client'; + +import { useEffect, useState } from 'react'; + +export function isSaveShortcut(event: KeyboardEvent): boolean { + if (event.isComposing || event.repeat) return false; + if (!(event.metaKey || event.ctrlKey)) return false; + if (event.key.toLowerCase() !== 's') return false; + if (document.querySelector('[role="alertdialog"][data-state="open"]')) { + return false; + } + return true; +} + +export function useSaveShortcut(enabled: boolean, onSave: () => void): void { + useEffect(() => { + if (!enabled) return; + const onKeyDown = (event: KeyboardEvent) => { + if (!isSaveShortcut(event)) return; + event.preventDefault(); + onSave(); + }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [enabled, onSave]); +} + +export function useSaveShortcutLabel(): string { + const [label, setLabel] = useState('Ctrl+S'); + useEffect(() => { + const platform = navigator.userAgent; + setLabel(/Mac|iPhone|iPad|iPod/i.test(platform) ? '⌘S' : 'Ctrl+S'); + }, []); + return label; +} diff --git a/src/lib/api/embeddings/indexes.ts b/src/lib/api/embeddings/indexes.ts new file mode 100644 index 000000000..5625dd8e7 --- /dev/null +++ b/src/lib/api/embeddings/indexes.ts @@ -0,0 +1,36 @@ +'use server'; + +import { getSchemas, getSchemaVectorIndexes } from '@/lib/api/database'; +import { SchemaIndexLookup } from '@/lib/models/embeddings'; + +export async function resolveIndexesBySchema( + schemaNames: Iterable +): Promise> { + const names = [...new Set(schemaNames)].filter(name => name.length > 0); + if (names.length === 0) return {}; + + let schemas: { name: string; _id: string }[] | undefined; + try { + const response = await getSchemas({ limit: 1000 }); + schemas = response.schemas; + } catch { + return Object.fromEntries(names.map(name => [name, 'unknown'])); + } + + const idByName = new Map(schemas.map(schema => [schema.name, schema._id])); + const lookups = await Promise.allSettled( + names.map(async (name): Promise<[string, SchemaIndexLookup]> => { + const schemaId = idByName.get(name); + if (!schemaId) return [name, []]; + const { indexes } = await getSchemaVectorIndexes(schemaId); + return [name, indexes]; + }) + ); + + const map: Record = {}; + for (const [index, name] of names.entries()) { + const result = lookups[index]; + map[name] = result.status === 'fulfilled' ? result.value[1] : 'unknown'; + } + return map; +} diff --git a/src/lib/models/embeddings/config-change.test.ts b/src/lib/models/embeddings/config-change.test.ts new file mode 100644 index 000000000..097db62a8 --- /dev/null +++ b/src/lib/models/embeddings/config-change.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + diffMaterialEmbeddingConfig, + isInPlaceDimensionChange, + requiresIndexRecreation, +} from './config-change.ts'; + +const base = { + provider: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 1536, + sourceFields: ['title', 'body'], + targetField: 'embedding', + similarity: 'cosine', +}; + +describe('material embedding config changes', () => { + it('treats source field order as equivalent', () => { + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { + ...base, + sourceFields: ['body', 'title'], + }), + [] + ); + }); + + it('detects provider, model, source, target, similarity, and dimension edits', () => { + assert.deepEqual( + diffMaterialEmbeddingConfig(base, { + provider: 'other', + model: 'text-embedding-3-large', + dimensions: 768, + sourceFields: ['title'], + targetField: 'vector', + similarity: 'euclidean', + }), + [ + 'provider', + 'model', + 'dimensions', + 'sourceFields', + 'targetField', + 'similarity', + ] + ); + }); + + it('requires index recreation only for dimensions, target, and similarity', () => { + assert.equal(requiresIndexRecreation(['sourceFields', 'provider']), false); + assert.equal(requiresIndexRecreation(['dimensions']), true); + assert.equal(requiresIndexRecreation(['targetField']), true); + assert.equal(requiresIndexRecreation(['similarity']), true); + }); + + it('flags in-place dimension changes on the same target field', () => { + assert.equal( + isInPlaceDimensionChange(base, { ...base, dimensions: 768 }), + true + ); + assert.equal( + isInPlaceDimensionChange(base, { + ...base, + dimensions: 768, + targetField: 'vector', + }), + false + ); + }); +}); diff --git a/src/lib/models/embeddings/config-change.ts b/src/lib/models/embeddings/config-change.ts new file mode 100644 index 000000000..31fff04b5 --- /dev/null +++ b/src/lib/models/embeddings/config-change.ts @@ -0,0 +1,84 @@ +export const MATERIAL_EMBEDDING_CONFIG_FIELDS = [ + 'provider', + 'model', + 'dimensions', + 'sourceFields', + 'targetField', + 'similarity', +] as const; + +export type MaterialEmbeddingConfigField = + (typeof MATERIAL_EMBEDDING_CONFIG_FIELDS)[number]; + +export type MaterialEmbeddingConfig = { + provider: string; + model: string; + dimensions: number; + sourceFields: readonly string[]; + targetField: string; + similarity: string; +}; + +export function normalizeSourceFields(fields: readonly string[]): string[] { + return [...fields].map(field => field.trim()).sort(); +} + +export function sameSourceFields( + left: readonly string[] | undefined, + right: readonly string[] | undefined +): boolean { + return ( + JSON.stringify(normalizeSourceFields(left ?? [])) === + JSON.stringify(normalizeSourceFields(right ?? [])) + ); +} + +export function toMaterialEmbeddingConfig( + config: MaterialEmbeddingConfig +): MaterialEmbeddingConfig { + return { + provider: config.provider, + model: config.model, + dimensions: config.dimensions, + sourceFields: config.sourceFields, + targetField: config.targetField, + similarity: config.similarity, + }; +} + +export function diffMaterialEmbeddingConfig( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig +): MaterialEmbeddingConfigField[] { + const changed: MaterialEmbeddingConfigField[] = []; + if (existing.provider !== next.provider) changed.push('provider'); + if (existing.model !== next.model) changed.push('model'); + if (existing.dimensions !== next.dimensions) changed.push('dimensions'); + if (!sameSourceFields(existing.sourceFields, next.sourceFields)) { + changed.push('sourceFields'); + } + if (existing.targetField !== next.targetField) changed.push('targetField'); + if (existing.similarity !== next.similarity) changed.push('similarity'); + return changed; +} + +export function requiresIndexRecreation( + changed: readonly MaterialEmbeddingConfigField[] +): boolean { + return changed.some( + field => + field === 'dimensions' || + field === 'targetField' || + field === 'similarity' + ); +} + +export function isInPlaceDimensionChange( + existing: MaterialEmbeddingConfig, + next: MaterialEmbeddingConfig +): boolean { + return ( + existing.targetField === next.targetField && + existing.dimensions !== next.dimensions + ); +} diff --git a/src/lib/models/embeddings/errors.ts b/src/lib/models/embeddings/errors.ts index e319b1736..3cba76284 100644 --- a/src/lib/models/embeddings/errors.ts +++ b/src/lib/models/embeddings/errors.ts @@ -30,6 +30,11 @@ function readBackendMessage(data: unknown): string | undefined { return undefined; } +export function isEmbeddingsNotFound(err: unknown): boolean { + if (!isAxiosLikeError(err)) return false; + return err.response?.status === 404; +} + export function formatEmbeddingsApiError(err: unknown): string { if (isAxiosLikeError(err)) { return ( diff --git a/src/lib/models/embeddings/index-state.ts b/src/lib/models/embeddings/index-state.ts new file mode 100644 index 000000000..9f2636f89 --- /dev/null +++ b/src/lib/models/embeddings/index-state.ts @@ -0,0 +1,133 @@ +import { BackfillRun } from './backfill'; +import { + isVectorIndexQueryable, + VectorCapabilities, + VectorIndexDefinition, +} from './capabilities'; +import { EmbeddingConfig, VectorSimilarity } from './config'; +import { + findMatchingIndex, + isVectorStorageSearchReady, + SchemaIndexLookup, +} from './readiness'; + +export const CONFIG_INDEX_STATES = [ + 'ready', + 'pending', + 'failed', + 'missing', + 'unknown', +] as const; + +export type ConfigIndexState = (typeof CONFIG_INDEX_STATES)[number]; + +export type EmbeddingConfigListRow = { + config: EmbeddingConfig; + indexState: ConfigIndexState; + latestBackfill?: BackfillRun; +}; + +export function resolveConfigIndexState( + config: EmbeddingConfig, + lookup: SchemaIndexLookup | undefined +): ConfigIndexState { + if (lookup == null || lookup === 'unknown') return 'unknown'; + const match = findMatchingIndex(config, lookup); + if (!match) return 'missing'; + if (match.status === 'failed') return 'failed'; + if (isVectorIndexQueryable(match)) return 'ready'; + return 'pending'; +} + +export function configIndexStateLabel(state: ConfigIndexState): string { + switch (state) { + case 'ready': + return 'Ready'; + case 'pending': + return 'Pending'; + case 'failed': + return 'Failed'; + case 'missing': + return 'Missing'; + case 'unknown': + return 'Unknown'; + default: { + const exhaustive: never = state; + return exhaustive; + } + } +} + +export function similarityLabel(value: VectorSimilarity): string { + switch (value) { + case 'cosine': + return 'Cosine'; + case 'euclidean': + return 'Euclidean'; + case 'dotProduct': + return 'Dot product'; + default: { + const exhaustive: never = value; + return exhaustive; + } + } +} + +export function parseIndexGeneration(name?: string): { + base: string; + generation: number; +} { + if (!name) return { base: '', generation: 0 }; + const match = /^(.*)_v(\d+)$/.exec(name); + if (match) { + return { base: match[1], generation: Number(match[2]) }; + } + return { base: name, generation: 1 }; +} + +export function canEnableEmbeddingConfig(args: { + capabilities?: VectorCapabilities; + matchingIndex?: VectorIndexDefinition; +}): boolean { + return ( + isVectorStorageSearchReady(args.capabilities) === true && + isVectorIndexQueryable(args.matchingIndex) + ); +} + +function backfillTimestamp(run: BackfillRun): number { + const raw = run.updatedAt ?? run.startedAt ?? run.createdAt; + if (!raw) return 0; + const time = Date.parse(raw); + return Number.isFinite(time) ? time : 0; +} + +export function pickLatestBackfill( + runs: BackfillRun[], + config: EmbeddingConfig +): BackfillRun | undefined { + const matched = runs.filter( + run => + run.configId === config._id || + (!run.configId && run.schemaName === config.schemaName) + ); + if (matched.length === 0) return undefined; + return matched.reduce((latest, run) => + backfillTimestamp(run) >= backfillTimestamp(latest) ? run : latest + ); +} + +export function buildConfigListRows( + configs: EmbeddingConfig[], + indexesBySchema: Record, + runs: BackfillRun[] +): EmbeddingConfigListRow[] { + return configs.map(config => ({ + config, + indexState: resolveConfigIndexState( + config, + indexesBySchema[config.schemaName] + ), + latestBackfill: pickLatestBackfill(runs, config), + })); +} diff --git a/src/lib/models/embeddings/index.ts b/src/lib/models/embeddings/index.ts index 8f6b89bed..d6f5592ac 100644 --- a/src/lib/models/embeddings/index.ts +++ b/src/lib/models/embeddings/index.ts @@ -7,3 +7,6 @@ export * from './errors'; export * from './secrets'; export * from './normalize'; export * from './readiness'; +export * from './source-fields'; +export * from './config-change'; +export * from './index-state'; diff --git a/src/lib/models/embeddings/source-fields.test.ts b/src/lib/models/embeddings/source-fields.test.ts new file mode 100644 index 000000000..a4fa86140 --- /dev/null +++ b/src/lib/models/embeddings/source-fields.test.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + isDeniedEmbeddingSchema, + isEligibleSourceField, + isHiddenField, + isSensitiveFieldName, + isStringLikeField, + listEligibleSchemas, + listEligibleSourceFields, + listSourceFieldChoices, + toSchemaFieldMap, +} from './source-fields.ts'; + +describe('source field eligibility', () => { + it('accepts string-like definitions and rejects hidden or sensitive names', () => { + assert.equal(isStringLikeField('String'), true); + assert.equal(isStringLikeField({ type: 'String' }), true); + assert.equal(isStringLikeField(['String']), true); + assert.equal(isStringLikeField('Number'), false); + assert.equal(isHiddenField({ type: 'String', select: false }), true); + assert.equal(isSensitiveFieldName('apiKey'), true); + assert.equal(isEligibleSourceField('title', { type: 'String' }), true); + assert.equal(isEligibleSourceField('password', { type: 'String' }), false); + assert.equal( + isEligibleSourceField('notes', { type: 'String', select: false }), + false + ); + }); + + it('lists eligible fields and keeps currently selected ineligible names', () => { + const fields = { + title: { type: 'String' }, + body: { type: 'String' }, + password: { type: 'String' }, + hidden: { type: 'String', select: false }, + count: { type: 'Number' }, + }; + assert.deepEqual(listEligibleSourceFields(fields), ['body', 'title']); + assert.deepEqual( + listSourceFieldChoices(fields, ['title', 'password']).map( + choice => choice.name + ), + ['body', 'password', 'title'] + ); + }); + + it('denies embeddings-owned, system, and auth-secret schemas', () => { + assert.equal( + isDeniedEmbeddingSchema({ name: 'Article', ownerModule: 'database' }), + false + ); + assert.equal( + isDeniedEmbeddingSchema({ + name: 'EmbeddingConfig', + ownerModule: 'embeddings', + }), + true + ); + assert.equal(isDeniedEmbeddingSchema({ name: 'Config' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: 'AccessToken' }), true); + assert.equal(isDeniedEmbeddingSchema({ name: '_internal' }), true); + }); + + it('prefers compiled fields and filters denied schemas', () => { + const fields = toSchemaFieldMap({ + fields: { title: { type: 'String' } }, + compiledFields: { body: { type: 'String' } }, + }); + assert.deepEqual(Object.keys(fields), ['body']); + const eligible = listEligibleSchemas([ + { + name: 'Article', + ownerModule: 'database', + fields: { title: { type: 'String' } }, + }, + { + name: 'EmbeddingConfig', + ownerModule: 'embeddings', + fields: { title: { type: 'String' } }, + }, + ]); + assert.deepEqual( + eligible.map(schema => schema.name), + ['Article'] + ); + }); +}); diff --git a/src/lib/models/embeddings/source-fields.ts b/src/lib/models/embeddings/source-fields.ts new file mode 100644 index 000000000..21ed5f8d9 --- /dev/null +++ b/src/lib/models/embeddings/source-fields.ts @@ -0,0 +1,157 @@ +export const AUTH_SECRET_SCHEMA_NAMES = new Set([ + 'AccessToken', + 'RefreshToken', + 'Token', + 'TwoFactorSecret', + 'TwoFactorBackUpCodes', + 'BiometricToken', + 'AdminTwoFactorSecret', + 'AdminApiToken', +]); + +export const SYSTEM_SCHEMA_NAMES = new Set([ + 'Views', + 'Config', + 'MigratedSchemas', + 'PendingSchemas', + 'CustomEndpoints', +]); + +export const EMBEDDINGS_OWNER_MODULE = 'embeddings'; + +export const EMBEDDING_OWNED_SCHEMA_NAMES = new Set([ + 'EmbeddingConfig', + 'BackfillRun', +]); + +const SENSITIVE_FIELD_NAME = + /(password|secret|token|credential|apikey|api_key|private[_-]?key|authorization|refresh[_-]?token|access[_-]?token)/i; + +const SOURCE_FIELD_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const SCHEMA_OR_TARGET_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; + +export type EmbeddingSchemaChoice = { + name: string; + ownerModule: string; + fields: Record; +}; + +export type SourceFieldChoice = { + name: string; + eligible: boolean; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isSensitiveFieldName(field: string): boolean { + return SENSITIVE_FIELD_NAME.test(field); +} + +export function isStringLikeField(field: unknown): boolean { + if (field === 'String') return true; + if (Array.isArray(field) && field.length === 1) { + return isStringLikeField(field[0]); + } + if (!isRecord(field)) return false; + if (field.type === 'String') return true; + return Array.isArray(field.type) && isStringLikeField(field.type); +} + +export function isHiddenField(field: unknown): boolean { + return isRecord(field) && field.select === false; +} + +export function isValidSourceFieldName(field: string): boolean { + return SOURCE_FIELD_NAME.test(field); +} + +export function isValidSchemaOrTargetName(value: string): boolean { + return SCHEMA_OR_TARGET_NAME.test(value); +} + +export function isDeniedEmbeddingSchema(schema: { + name: string; + ownerModule?: string; +}): boolean { + if (!schema.name) return true; + if (schema.ownerModule === EMBEDDINGS_OWNER_MODULE) return true; + if (EMBEDDING_OWNED_SCHEMA_NAMES.has(schema.name)) return true; + if (schema.name.startsWith('_')) return true; + if (SYSTEM_SCHEMA_NAMES.has(schema.name)) return true; + return AUTH_SECRET_SCHEMA_NAMES.has(schema.name); +} + +export function toSchemaFieldMap(schema: { + fields?: unknown; + compiledFields?: unknown; +}): Record { + if ( + isRecord(schema.compiledFields) && + Object.keys(schema.compiledFields).length > 0 + ) { + return schema.compiledFields; + } + if (isRecord(schema.fields)) return schema.fields; + return {}; +} + +export function isEligibleSourceField( + name: string, + definition: unknown +): boolean { + if (!isValidSourceFieldName(name)) return false; + if (!isStringLikeField(definition)) return false; + if (isHiddenField(definition)) return false; + return !isSensitiveFieldName(name); +} + +export function listEligibleSourceFields( + fields: Record +): string[] { + return Object.keys(fields) + .filter(name => isEligibleSourceField(name, fields[name])) + .sort(); +} + +export function listSourceFieldChoices( + fields: Record, + current: readonly string[] +): SourceFieldChoice[] { + const names = new Set([ + ...listEligibleSourceFields(fields), + ...current.filter(name => name.length > 0), + ]); + return [...names].sort().map(name => ({ + name, + eligible: isEligibleSourceField(name, fields[name]), + })); +} + +export function toEmbeddingSchemaChoice(schema: { + name: string; + ownerModule: string; + fields?: unknown; + compiledFields?: unknown; +}): EmbeddingSchemaChoice { + return { + name: schema.name, + ownerModule: schema.ownerModule, + fields: toSchemaFieldMap(schema), + }; +} + +export function listEligibleSchemas( + schemas: Array<{ + name: string; + ownerModule: string; + fields?: unknown; + compiledFields?: unknown; + }> +): EmbeddingSchemaChoice[] { + return schemas + .filter(schema => !isDeniedEmbeddingSchema(schema)) + .map(toEmbeddingSchemaChoice) + .sort((left, right) => left.name.localeCompare(right.name)); +} From ee5e432a7a6510510f91d54c3d83a1d1d6669a3f Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Tue, 8 Sep 2026 21:32:14 +0300 Subject: [PATCH 04/24] feat(embeddings): add provider settings and worker toggle Save provider settings without submitting a redacted API key. Enable workers separately from the deployed embeddings workload. --- .../(modules)/embeddings/settings/loading.tsx | 15 ++ .../(modules)/embeddings/settings/page.tsx | 57 ++++++ .../embeddings/configs/use-save-shortcut.ts | 22 ++- .../embeddings/settings/advanced-settings.tsx | 161 ++++++++++++++++ .../settings/embeddings-settings.tsx | 177 ++++++++++++++++++ .../embeddings/settings/hosts-field.tsx | 130 +++++++++++++ .../embeddings/settings/schema.test.ts | 73 ++++++++ src/components/embeddings/settings/schema.ts | 124 ++++++++++++ .../embeddings/settings/settings-form.tsx | 166 ++++++++++++++++ .../settings/SettingsFormActions.tsx | 45 ++++- src/lib/models/embeddings/index.ts | 1 + src/lib/models/embeddings/secrets.test.ts | 65 +++++++ src/lib/models/embeddings/secrets.ts | 9 +- .../models/embeddings/settings-form.test.ts | 102 ++++++++++ src/lib/models/embeddings/settings-form.ts | 149 +++++++++++++++ 15 files changed, 1287 insertions(+), 9 deletions(-) create mode 100644 src/app/(dashboard)/(modules)/embeddings/settings/loading.tsx create mode 100644 src/app/(dashboard)/(modules)/embeddings/settings/page.tsx create mode 100644 src/components/embeddings/settings/advanced-settings.tsx create mode 100644 src/components/embeddings/settings/embeddings-settings.tsx create mode 100644 src/components/embeddings/settings/hosts-field.tsx create mode 100644 src/components/embeddings/settings/schema.test.ts create mode 100644 src/components/embeddings/settings/schema.ts create mode 100644 src/components/embeddings/settings/settings-form.tsx create mode 100644 src/lib/models/embeddings/secrets.test.ts create mode 100644 src/lib/models/embeddings/settings-form.test.ts create mode 100644 src/lib/models/embeddings/settings-form.ts diff --git a/src/app/(dashboard)/(modules)/embeddings/settings/loading.tsx b/src/app/(dashboard)/(modules)/embeddings/settings/loading.tsx new file mode 100644 index 000000000..ea624c102 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/settings/loading.tsx @@ -0,0 +1,15 @@ +import { Skeleton } from '@/components/ui/skeleton'; + +export default function EmbeddingsSettingsLoading() { + return ( +
+
+ + +
+ + + +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/settings/page.tsx b/src/app/(dashboard)/(modules)/embeddings/settings/page.tsx new file mode 100644 index 000000000..a0fa6d88b --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/settings/page.tsx @@ -0,0 +1,57 @@ +import Link from 'next/link'; +import { EmbeddingsSettings } from '@/components/embeddings/settings/embeddings-settings'; +import { ErrorCard } from '@/components/error/ErrorCard'; +import { Button } from '@/components/ui/button'; +import { + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { getEmbeddingsSettings } from '@/lib/api/embeddings'; +import { getModules } from '@/lib/api/modules'; +import { isModuleServing } from '@/lib/api/modules/patch-settings-options'; +import { formatEmbeddingsApiError } from '@/lib/models/embeddings/errors'; + +export default async function EmbeddingsSettingsPage() { + const [settingsResult, modulesResult] = await Promise.allSettled([ + getEmbeddingsSettings(), + getModules(), + ]); + + if (settingsResult.status === 'rejected') { + return ( + + Embeddings overview + + } + /> + ); + } + + const serving = + modulesResult.status === 'fulfilled' + ? isModuleServing(modulesResult.value, 'embeddings') + : undefined; + + return ( +
+ +
+ Settings + + Provider credentials, allowed hosts, workers, and operational + limits. + +
+ + +
+ ); +} diff --git a/src/components/embeddings/configs/use-save-shortcut.ts b/src/components/embeddings/configs/use-save-shortcut.ts index 0d5123e34..513a12c4e 100644 --- a/src/components/embeddings/configs/use-save-shortcut.ts +++ b/src/components/embeddings/configs/use-save-shortcut.ts @@ -2,13 +2,27 @@ import { useEffect, useState } from 'react'; +export function isShortcutOverlayOpen(): boolean { + return Boolean( + document.querySelector('[role="alertdialog"][data-state="open"]') || + document.querySelector('[role="dialog"][data-state="open"]') + ); +} + export function isSaveShortcut(event: KeyboardEvent): boolean { - if (event.isComposing || event.repeat) return false; + if (event.isComposing || event.repeat || event.defaultPrevented) return false; if (!(event.metaKey || event.ctrlKey)) return false; + if (event.altKey) return false; if (event.key.toLowerCase() !== 's') return false; - if (document.querySelector('[role="alertdialog"][data-state="open"]')) { - return false; - } + if (isShortcutOverlayOpen()) return false; + return true; +} + +export function isCancelShortcut(event: KeyboardEvent): boolean { + if (event.isComposing || event.repeat || event.defaultPrevented) return false; + if (event.key !== 'Escape') return false; + if (event.metaKey || event.ctrlKey || event.altKey) return false; + if (isShortcutOverlayOpen()) return false; return true; } diff --git a/src/components/embeddings/settings/advanced-settings.tsx b/src/components/embeddings/settings/advanced-settings.tsx new file mode 100644 index 000000000..839023cae --- /dev/null +++ b/src/components/embeddings/settings/advanced-settings.tsx @@ -0,0 +1,161 @@ +'use client'; + +import { ChevronDown } from 'lucide-react'; +import { InputField } from '@/components/ui/form-inputs/InputField'; +import SwitchField from '@/components/ui/form-inputs/SwitchField'; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from '@/components/ui/collapsible'; +import { HostsField } from '@/components/embeddings/settings/hosts-field'; +import { SETTINGS_LIMITS } from '@/lib/models/embeddings/settings-form'; + +type AdvancedSettingsProps = { + disabled: boolean; +}; + +export function AdvancedSettings({ disabled }: AdvancedSettingsProps) { + return ( + + + Advanced limits + + + +
+
+

+ Queue +

+
+ + + + +
+
+
+

+ Security +

+
+ +

+ Always enforced when the server runs in production. +

+
+ +
+
+

+ Input and event limits +

+
+ + + + +
+
+
+ + + ); +} diff --git a/src/components/embeddings/settings/embeddings-settings.tsx b/src/components/embeddings/settings/embeddings-settings.tsx new file mode 100644 index 000000000..b383059a3 --- /dev/null +++ b/src/components/embeddings/settings/embeddings-settings.tsx @@ -0,0 +1,177 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; +import { useForm } from 'react-hook-form'; +import { useRouter } from 'next/navigation'; +import { Form } from '@/components/ui/form'; +import { ModuleToggle } from '@/components/settings/ModuleToggle'; +import { SettingsForm } from '@/components/embeddings/settings/settings-form'; +import { embeddingsSettingsFormSchema } from '@/components/embeddings/settings/schema'; +import { + isCancelShortcut, + useSaveShortcut, + useSaveShortcutLabel, +} from '@/components/embeddings/configs/use-save-shortcut'; +import { useAlerts } from '@/components/providers/AlertProvider'; +import { + activeTogglePatchOptions, + useSettingsSave, +} from '@/lib/hooks/use-settings-save'; +import { patchEmbeddingsSettings } from '@/lib/api/embeddings'; +import { + isModuleServing, + PatchSettingsResult, +} from '@/lib/api/modules/patch-settings-options'; +import type { EmbeddingsSettings as EmbeddingsModuleSettings } from '@/lib/models/embeddings/settings'; +import { + toSettingsFormValues, + toSettingsPatch, +} from '@/lib/models/embeddings/settings-form'; +import { rhfZodResolver } from '@/lib/zod-form'; +import { cn } from '@/lib/utils'; + +type EmbeddingsSettingsProps = { + data: EmbeddingsModuleSettings; + serving?: boolean; +}; + +function isWorkersPatchSuccess(result: PatchSettingsResult | void) { + if (!result) return true; + return isModuleServing(result.modules, 'embeddings'); +} + +export function EmbeddingsSettings({ data, serving }: EmbeddingsSettingsProps) { + const router = useRouter(); + const { addAlert } = useAlerts(); + const { save, isSaving } = useSettingsSave('Embeddings'); + const shortcutLabel = useSaveShortcutLabel(); + const [workersEnabled, setWorkersEnabled] = useState(data.enabled); + const [edit, setEdit] = useState(false); + const form = useForm({ + resolver: rhfZodResolver(embeddingsSettingsFormSchema), + mode: 'onChange', + defaultValues: toSettingsFormValues(data), + }); + const { handleSubmit, reset, formState } = form; + const dirty = formState.isDirty; + + useEffect(() => { + setWorkersEnabled(data.enabled); + reset(toSettingsFormValues(data)); + }, [data, reset]); + + const onSubmit = useCallback( + async (values: ReturnType) => { + const result = await save({ + action: () => + patchEmbeddingsSettings( + toSettingsPatch(values, data, workersEnabled) + ), + onSuccess: () => router.refresh(), + }); + if (result.ok) { + setEdit(false); + } + }, + [data, router, save, workersEnabled] + ); + + const cancelEdit = useCallback(() => { + reset(toSettingsFormValues(data)); + setEdit(false); + }, [data, reset]); + + useSaveShortcut( + edit && dirty && !isSaving, + useCallback(() => { + void handleSubmit(onSubmit)(); + }, [handleSubmit, onSubmit]) + ); + + useEffect(() => { + if (!edit) return; + const onKeyDown = (event: KeyboardEvent) => { + if (!isCancelShortcut(event)) return; + event.preventDefault(); + cancelEdit(); + }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [cancelEdit, edit]); + + const handleWorkersToggle = () => { + const nextEnabled = !workersEnabled; + addAlert({ + title: nextEnabled ? 'Enable workers' : 'Disable workers', + description: nextEnabled + ? serving === false + ? 'The embeddings workload is not serving. Workers will not process jobs until the module is deployed. Enable workers anyway?' + : 'Enable generation workers and mutation subscriptions? This does not deploy or stop the embeddings workload.' + : 'Disable generation workers and mutation subscriptions? The embeddings workload stays deployed.', + cancelText: 'Cancel', + actionText: 'Proceed', + onDecision: cancel => { + if (cancel) return; + setWorkersEnabled(nextEnabled); + void save({ + action: () => + patchEmbeddingsSettings( + { enabled: nextEnabled }, + nextEnabled + ? activeTogglePatchOptions(['embeddings'], true) + : undefined + ), + isActivationSuccess: nextEnabled ? isWorkersPatchSuccess : undefined, + onSuccess: () => router.refresh(), + onError: () => setWorkersEnabled(data.enabled), + onActivationFailure: () => setWorkersEnabled(data.enabled), + }); + }, + }); + }; + + return ( +
+
+ +

+ Workers process embedding jobs and subscribe to schema mutations. The + toggle does not deploy or remove the embeddings service. +

+

+ {serving === true + ? 'Workload is serving. Workers can run when enabled.' + : serving === false + ? 'Workload is not serving. Deploy the embeddings module before jobs can run.' + : 'Workload serving state is unavailable.'} +

+
+
+ undefined)}> + + + +
+ ); +} diff --git a/src/components/embeddings/settings/hosts-field.tsx b/src/components/embeddings/settings/hosts-field.tsx new file mode 100644 index 000000000..389759f02 --- /dev/null +++ b/src/components/embeddings/settings/hosts-field.tsx @@ -0,0 +1,130 @@ +'use client'; + +import { KeyboardEvent, useState } from 'react'; +import { X } from 'lucide-react'; +import { useFormContext } from 'react-hook-form'; +import { Button } from '@/components/ui/button'; +import { + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { cn } from '@/lib/utils'; +import { normalizeHost } from '@/lib/models/embeddings/settings-form'; + +type HostsFieldName = 'allowedHosts' | 'security.sourceFieldAllowlist'; + +type HostsFieldProps = { + name: HostsFieldName; + label: string; + description: string; + disabled?: boolean; + placeholder?: string; +}; + +export function HostsField({ + name, + label, + description, + disabled = false, + placeholder = 'Add a host and press Enter', +}: HostsFieldProps) { + const { control } = useFormContext(); + const [draft, setDraft] = useState(''); + + return ( + { + const hosts = Array.isArray(field.value) + ? field.value.filter(item => typeof item === 'string') + : []; + + const commit = (raw: string) => { + const host = normalizeHost(raw); + if (!host) return; + if (hosts.includes(host)) { + setDraft(''); + return; + } + field.onChange([...hosts, host]); + setDraft(''); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Enter' || event.key === ',' || event.key === ' ') { + if (!draft.trim()) return; + event.preventDefault(); + commit(draft); + return; + } + if (event.key === 'Escape' && draft) { + event.preventDefault(); + event.stopPropagation(); + setDraft(''); + return; + } + if (event.key === 'Backspace' && !draft && hosts.length > 0) { + field.onChange(hosts.slice(0, -1)); + } + }; + + return ( + + {label} + +
+ {hosts.map(host => ( + + {host} + + + ))} + setDraft(event.target.value)} + onKeyDown={onKeyDown} + onBlur={() => { + if (draft.trim()) commit(draft); + field.onBlur(); + }} + disabled={disabled} + placeholder={hosts.length === 0 ? placeholder : undefined} + autoComplete="off" + className="h-8 min-w-40 flex-1 border-0 bg-transparent px-1 shadow-none focus-visible:ring-0" + /> +
+ + {description} + + + ); + }} + /> + ); +} diff --git a/src/components/embeddings/settings/schema.test.ts b/src/components/embeddings/settings/schema.test.ts new file mode 100644 index 000000000..05181ed78 --- /dev/null +++ b/src/components/embeddings/settings/schema.test.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { REDACTED_SECRET } from '../../../lib/models/embeddings/secrets.ts'; +import { OPENAI_COMPATIBLE_PROVIDER } from '../../../lib/models/embeddings/settings.ts'; +import { embeddingsSettingsFormSchema } from './schema.ts'; + +const valid = { + defaultProvider: OPENAI_COMPATIBLE_PROVIDER, + endpoint: 'https://api.openai.com/v1/embeddings', + apiKey: '', + apiKeyConfigured: true, + model: 'text-embedding-3-small', + allowedHosts: ['api.openai.com', ' API.OpenAI.com '], + queue: { + concurrency: 2, + attempts: 3, + maxBatchSize: 500, + drainTimeoutMs: 15 * 60 * 1000, + }, + security: { + requireGrpcKey: false, + sourceFieldAllowlist: ['title'], + maxMutationEventIds: 500, + embedTimeoutMs: 10_000, + maxEmbedInputBytes: 32 * 1024, + maxEmbedResponseBytes: 1024 * 1024, + }, +}; + +describe('embeddings settings schema', () => { + it('accepts a redacted-configured key and normalizes hosts', () => { + const parsed = embeddingsSettingsFormSchema.parse(valid); + assert.deepEqual(parsed.allowedHosts, ['api.openai.com']); + assert.equal(parsed.apiKey, ''); + }); + + it('requires a key when none is stored and requires HTTPS', () => { + const missingKey = embeddingsSettingsFormSchema.safeParse({ + ...valid, + apiKeyConfigured: false, + apiKey: '', + }); + assert.equal(missingKey.success, false); + const http = embeddingsSettingsFormSchema.safeParse({ + ...valid, + endpoint: 'http://api.openai.com/v1/embeddings', + }); + assert.equal(http.success, false); + }); + + it('blanks a redaction marker instead of treating it as a new key', () => { + const parsed = embeddingsSettingsFormSchema.parse({ + ...valid, + apiKey: REDACTED_SECRET, + apiKeyConfigured: true, + }); + assert.equal(parsed.apiKey, ''); + }); + + it('requires the endpoint host and rejects out-of-range numbers', () => { + const missingHost = embeddingsSettingsFormSchema.safeParse({ + ...valid, + allowedHosts: ['example.com'], + }); + assert.equal(missingHost.success, false); + const concurrency = embeddingsSettingsFormSchema.safeParse({ + ...valid, + queue: { ...valid.queue, concurrency: 0 }, + }); + assert.equal(concurrency.success, false); + }); +}); diff --git a/src/components/embeddings/settings/schema.ts b/src/components/embeddings/settings/schema.ts new file mode 100644 index 000000000..fff0454ff --- /dev/null +++ b/src/components/embeddings/settings/schema.ts @@ -0,0 +1,124 @@ +import { z } from 'zod'; +import { formApiKeyValue } from '../../../lib/models/embeddings/secrets.ts'; +import { OPENAI_COMPATIBLE_PROVIDER } from '../../../lib/models/embeddings/settings.ts'; +import { + isValidHost, + normalizeHosts, + parseHttpsEndpoint, + SETTINGS_LIMITS, +} from '../../../lib/models/embeddings/settings-form.ts'; + +function boundedInt(limits: { min: number; max: number }, message: string) { + return z.coerce + .number({ error: message }) + .int(message) + .min(limits.min, message) + .max(limits.max, message); +} + +const hostListSchema = z + .array(z.string()) + .transform(normalizeHosts) + .superRefine((hosts, ctx) => { + if (hosts.some(host => !isValidHost(host))) { + ctx.addIssue({ + code: 'custom', + message: 'Use a hostname such as api.openai.com', + }); + } + }); + +export const embeddingsSettingsFormSchema = z + .object({ + defaultProvider: z + .string() + .min(1, 'Provider is required') + .refine( + value => value === OPENAI_COMPATIBLE_PROVIDER, + 'Provider must be openai-compatible' + ), + endpoint: z + .string() + .trim() + .min(1, 'Endpoint is required') + .refine( + value => parseHttpsEndpoint(value) != null, + 'Endpoint must be HTTPS with no credentials' + ), + apiKey: z.string().transform(formApiKeyValue), + apiKeyConfigured: z.boolean(), + model: z.string().trim().min(1, 'Model is required'), + allowedHosts: hostListSchema, + queue: z.object({ + concurrency: boundedInt( + SETTINGS_LIMITS.concurrency, + `Concurrency must be ${SETTINGS_LIMITS.concurrency.min}–${SETTINGS_LIMITS.concurrency.max}` + ), + attempts: boundedInt( + SETTINGS_LIMITS.attempts, + `Attempts must be ${SETTINGS_LIMITS.attempts.min}–${SETTINGS_LIMITS.attempts.max}` + ), + maxBatchSize: boundedInt( + SETTINGS_LIMITS.maxBatchSize, + `Max batch size must be ${SETTINGS_LIMITS.maxBatchSize.min}–${SETTINGS_LIMITS.maxBatchSize.max}` + ), + drainTimeoutMs: boundedInt( + SETTINGS_LIMITS.drainTimeoutMs, + `Drain timeout must be ${SETTINGS_LIMITS.drainTimeoutMs.min}–${SETTINGS_LIMITS.drainTimeoutMs.max} ms` + ), + }), + security: z.object({ + requireGrpcKey: z.boolean(), + sourceFieldAllowlist: z + .array(z.string()) + .transform(values => + values.map(value => value.trim()).filter(value => value.length > 0) + ), + maxMutationEventIds: boundedInt( + SETTINGS_LIMITS.maxMutationEventIds, + `Max mutation event ids must be ${SETTINGS_LIMITS.maxMutationEventIds.min}–${SETTINGS_LIMITS.maxMutationEventIds.max}` + ), + embedTimeoutMs: boundedInt( + SETTINGS_LIMITS.embedTimeoutMs, + `Embed timeout must be ${SETTINGS_LIMITS.embedTimeoutMs.min}–${SETTINGS_LIMITS.embedTimeoutMs.max} ms` + ), + maxEmbedInputBytes: boundedInt( + SETTINGS_LIMITS.maxEmbedInputBytes, + `Max input bytes must be ${SETTINGS_LIMITS.maxEmbedInputBytes.min}–${SETTINGS_LIMITS.maxEmbedInputBytes.max}` + ), + maxEmbedResponseBytes: boundedInt( + SETTINGS_LIMITS.maxEmbedResponseBytes, + `Max response bytes must be ${SETTINGS_LIMITS.maxEmbedResponseBytes.min}–${SETTINGS_LIMITS.maxEmbedResponseBytes.max}` + ), + }), + }) + .superRefine((value, ctx) => { + if (!value.apiKeyConfigured && value.apiKey.trim().length === 0) { + ctx.addIssue({ + code: 'custom', + path: ['apiKey'], + message: 'API key is required', + }); + } + const url = parseHttpsEndpoint(value.endpoint); + if (!url) return; + const hostname = url.hostname.toLowerCase(); + if (!value.allowedHosts.includes(hostname)) { + ctx.addIssue({ + code: 'custom', + path: ['allowedHosts'], + message: `Allowlist must include ${hostname}`, + }); + } + if (value.allowedHosts.length === 0) { + ctx.addIssue({ + code: 'custom', + path: ['allowedHosts'], + message: 'Add at least one allowed host', + }); + } + }); + +export type EmbeddingsSettingsSchemaValues = z.infer< + typeof embeddingsSettingsFormSchema +>; diff --git a/src/components/embeddings/settings/settings-form.tsx b/src/components/embeddings/settings/settings-form.tsx new file mode 100644 index 000000000..9bcb5b02b --- /dev/null +++ b/src/components/embeddings/settings/settings-form.tsx @@ -0,0 +1,166 @@ +'use client'; + +import { useFormContext, useWatch } from 'react-hook-form'; +import { InputField } from '@/components/ui/form-inputs/InputField'; +import { + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { PasswordInput } from '@/components/ui/password-input'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { AdvancedSettings } from '@/components/embeddings/settings/advanced-settings'; +import { HostsField } from '@/components/embeddings/settings/hosts-field'; +import { SettingsFormActions } from '@/components/settings/SettingsFormActions'; +import { OPENAI_COMPATIBLE_PROVIDER } from '@/lib/models/embeddings/settings'; + +type SettingsFormProps = { + edit: boolean; + isSaving: boolean; + dirty: boolean; + shortcutLabel: string; + setEdit: (edit: boolean) => void; + onCancel: () => void; +}; + +export function SettingsForm({ + edit, + isSaving, + dirty, + shortcutLabel, + setEdit, + onCancel, +}: SettingsFormProps) { + const { control } = useFormContext(); + + return ( +
+
+
+

+ OpenAI-compatible provider +

+

+ Endpoint, stored API key, allowed hosts, and the default model used + when a config does not override them. +

+
+
+ ( + + Default provider + + + Currently the only supported provider. + + + + )} + /> + +
+ +
+ +
+ +
+
+
+ + setEdit(true)} + onCancel={onCancel} + /> +
+ ); +} + +function ApiKeyField({ disabled }: { disabled: boolean }) { + const { control } = useFormContext(); + const configured = useWatch({ control, name: 'apiKeyConfigured' }); + + return ( + ( + + API key + + + + + {configured + ? 'A key is already stored. Leave blank to keep it, or enter a replacement.' + : 'Required to call the provider. The stored value is never shown.'} + + + + )} + /> + ); +} diff --git a/src/components/settings/SettingsFormActions.tsx b/src/components/settings/SettingsFormActions.tsx index d20a38f47..9b40ccb81 100644 --- a/src/components/settings/SettingsFormActions.tsx +++ b/src/components/settings/SettingsFormActions.tsx @@ -1,10 +1,19 @@ import { Button } from '@/components/ui/button'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; interface SettingsFormActionsProps { edit: boolean; isSaving?: boolean; onEdit: () => void; onCancel: () => void; + shortcutLabel?: string; + dirty?: boolean; + submitLabel?: string; } export function SettingsFormActions({ @@ -12,7 +21,13 @@ export function SettingsFormActions({ isSaving = false, onEdit, onCancel, + shortcutLabel, + dirty, + submitLabel = 'Submit', }: SettingsFormActionsProps) { + const showShortcut = Boolean(edit && dirty && shortcutLabel && !isSaving); + const submitDisabled = isSaving || dirty === false; + return (
{edit ? ( @@ -26,9 +41,33 @@ export function SettingsFormActions({ > Cancel - + + + + + + +

+ {dirty === false + ? 'No unsaved changes' + : showShortcut + ? `Save settings ${shortcutLabel}` + : submitLabel} +

+ + +
) : ( + + + + ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/backfills/[id]/page.tsx b/src/app/(dashboard)/(modules)/embeddings/backfills/[id]/page.tsx new file mode 100644 index 000000000..d4977f95e --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/backfills/[id]/page.tsx @@ -0,0 +1,20 @@ +import { notFound } from 'next/navigation'; +import { BackfillDetail } from '@/components/embeddings/backfills/backfill-detail'; +import { getBackfill } from '@/lib/api/embeddings'; +import { BackfillRun, isEmbeddingsNotFound } from '@/lib/models/embeddings'; + +export default async function EmbeddingBackfillDetailPage(props: { + params: Promise<{ id: string }>; +}) { + const { id } = await props.params; + + let run: BackfillRun; + try { + run = await getBackfill(id); + } catch (error) { + if (isEmbeddingsNotFound(error)) notFound(); + throw error; + } + + return ; +} diff --git a/src/app/(dashboard)/(modules)/embeddings/backfills/loading.tsx b/src/app/(dashboard)/(modules)/embeddings/backfills/loading.tsx new file mode 100644 index 000000000..3a67954a5 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/backfills/loading.tsx @@ -0,0 +1,21 @@ +import { Skeleton } from '@/components/ui/skeleton'; + +export default function EmbeddingBackfillsLoading() { + return ( +
+
+
+ + +
+ +
+
+ + + +
+ +
+ ); +} diff --git a/src/app/(dashboard)/(modules)/embeddings/backfills/page.tsx b/src/app/(dashboard)/(modules)/embeddings/backfills/page.tsx new file mode 100644 index 000000000..58d0955a2 --- /dev/null +++ b/src/app/(dashboard)/(modules)/embeddings/backfills/page.tsx @@ -0,0 +1,144 @@ +import Link from 'next/link'; +import { Info } from 'lucide-react'; +import { BackfillsTable } from '@/components/embeddings/backfills/backfills-table'; +import { StartBackfillDialog } from '@/components/embeddings/backfills/start-backfill-dialog'; +import { ErrorCard } from '@/components/error/ErrorCard'; +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { Button } from '@/components/ui/button'; +import { + PageActions, + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { + getBackfills, + getEmbeddingConfigs, + getEmbeddingsSettings, + getEmbeddingsStatus, +} from '@/lib/api/embeddings'; +import { + formatEmbeddingsApiError, + mergeActiveBackfillRuns, + parseBackfillListParams, + toBackfillListQuery, + uniqueSchemaNames, +} from '@/lib/models/embeddings'; + +function settledValue(result: PromiseSettledResult): T | undefined { + return result.status === 'fulfilled' ? result.value : undefined; +} + +function settledError( + result: PromiseSettledResult +): string | undefined { + if (result.status !== 'rejected') return undefined; + return formatEmbeddingsApiError(result.reason); +} + +export default async function EmbeddingBackfillsPage(props: { + searchParams: Promise>; +}) { + const searchParams = await props.searchParams; + const query = parseBackfillListParams(searchParams); + const listQuery = toBackfillListQuery(query); + + const [ + listResult, + queuedResult, + runningResult, + configsResult, + settingsResult, + statusResult, + ] = await Promise.allSettled([ + getBackfills(listQuery), + query.state + ? Promise.resolve({ runs: [], count: 0 }) + : getBackfills({ ...listQuery, state: 'queued', skip: 0, limit: 100 }), + query.state + ? Promise.resolve({ runs: [], count: 0 }) + : getBackfills({ ...listQuery, state: 'running', skip: 0, limit: 100 }), + getEmbeddingConfigs(), + getEmbeddingsSettings(), + getEmbeddingsStatus(), + ]); + + const listError = settledError(listResult); + if (listError) { + return ( + + Embeddings overview + + } + /> + ); + } + + const list = settledValue(listResult) ?? { runs: [], count: 0 }; + const runs = mergeActiveBackfillRuns({ + pageRuns: list.runs, + queuedRuns: settledValue(queuedResult)?.runs ?? [], + runningRuns: settledValue(runningResult)?.runs ?? [], + skip: query.skip, + state: query.state, + }); + const configs = settledValue(configsResult) ?? []; + const schemas = uniqueSchemaNames(configs, runs); + if (query.schema && !schemas.includes(query.schema)) { + schemas.push(query.schema); + schemas.sort((left, right) => left.localeCompare(right)); + } + const settings = settledValue(settingsResult)?.config; + const status = settledValue(statusResult); + const workersEnabled = status?.enabled ?? settings?.enabled; + + return ( +
+ +
+ Backfills + + Filterable run history. Active runs appear first. + +
+ + + + + {workersEnabled === false ? ( + + + Workers disabled + + History stays available.{' '} + + Enable workers + {' '} + to process queued runs. + + + ) : null} + 0} + /> +
+ ); +} diff --git a/src/components/embeddings/backfills/backfill-actions.tsx b/src/components/embeddings/backfills/backfill-actions.tsx new file mode 100644 index 000000000..ab65a399b --- /dev/null +++ b/src/components/embeddings/backfills/backfill-actions.tsx @@ -0,0 +1,88 @@ +'use client'; + +import { useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { CancelBackfillDialog } from '@/components/embeddings/backfills/cancel-backfill-dialog'; +import { Button } from '@/components/ui/button'; +import { cancelBackfill, resumeBackfill } from '@/lib/api/embeddings'; +import { toast } from '@/lib/hooks/use-toast'; +import { + BackfillRun, + canCancelBackfill, + isResumeEligible, +} from '@/lib/models/embeddings/backfill'; +import { formatEmbeddingsApiError } from '@/lib/models/embeddings/errors'; + +type BackfillActionsProps = { + run: BackfillRun; +}; + +export function BackfillActions({ run }: BackfillActionsProps) { + const router = useRouter(); + const [cancelOpen, setCancelOpen] = useState(false); + const [pending, setPending] = useState(false); + const canCancel = canCancelBackfill(run.state); + const canResume = isResumeEligible(run.state); + + if (!canCancel && !canResume) return null; + + const onCancel = async () => { + setPending(true); + try { + await cancelBackfill(run._id); + toast({ title: 'Backfill canceled' }); + setCancelOpen(false); + router.refresh(); + } catch (error) { + toast({ + title: 'Could not cancel backfill', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + } finally { + setPending(false); + } + }; + + const onResume = async () => { + setPending(true); + try { + await resumeBackfill(run._id); + toast({ title: 'Backfill resumed' }); + router.refresh(); + } catch (error) { + toast({ + title: 'Could not resume backfill', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + } finally { + setPending(false); + } + }; + + return ( + <> + {canCancel ? ( + + ) : null} + {canResume ? ( + + ) : null} + void onCancel()} + /> + + ); +} diff --git a/src/components/embeddings/backfills/backfill-counters.tsx b/src/components/embeddings/backfills/backfill-counters.tsx new file mode 100644 index 000000000..0a2881316 --- /dev/null +++ b/src/components/embeddings/backfills/backfill-counters.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { BackfillRun } from '@/lib/models/embeddings/backfill'; +import { backfillProgress } from '@/lib/models/embeddings/backfill-view'; + +type BackfillCountersProps = { + run: BackfillRun; +}; + +function Count({ label, value }: { label: string; value: number }) { + return ( +
+

{label}

+

+ {value.toLocaleString()} +

+
+ ); +} + +export function BackfillCounters({ run }: BackfillCountersProps) { + const progress = backfillProgress(run); + return ( +
+
+ + + + +
+
+ {progress.percent != null ? ( +
+
+
+ ) : null} +

+ {progress.caption} +

+
+
+ ); +} diff --git a/src/components/embeddings/backfills/backfill-detail.tsx b/src/components/embeddings/backfills/backfill-detail.tsx new file mode 100644 index 000000000..9c89f185c --- /dev/null +++ b/src/components/embeddings/backfills/backfill-detail.tsx @@ -0,0 +1,155 @@ +'use client'; + +import Link from 'next/link'; +import { AlertTriangle } from 'lucide-react'; +import { BackfillActions } from '@/components/embeddings/backfills/backfill-actions'; +import { BackfillCounters } from '@/components/embeddings/backfills/backfill-counters'; +import { BackfillStateBadge } from '@/components/embeddings/backfills/backfill-state-badge'; +import { useBackfillPolling } from '@/components/embeddings/backfills/use-backfill-polling'; +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + PageActions, + PageDescription, + PageHeader, + PageTitle, +} from '@/components/ui/page-header'; +import { + BackfillRun, + isActiveBackfillState, +} from '@/lib/models/embeddings/backfill'; +import { + backfillTimeline, + formatBackfillTimestamp, + sanitizeBackfillErrorDisplay, +} from '@/lib/models/embeddings/backfill-view'; +import { cn } from '@/lib/utils'; + +type BackfillDetailProps = { + run: BackfillRun; +}; + +function Fact({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +export function BackfillDetail({ run }: BackfillDetailProps) { + useBackfillPolling(isActiveBackfillState(run.state)); + const timeline = backfillTimeline(run); + const error = sanitizeBackfillErrorDisplay(run.error); + const filterJson = + run.filter && Object.keys(run.filter).length > 0 + ? JSON.stringify(run.filter, null, 2) + : undefined; + + return ( +
+ +
+ {run.schemaName} + + Backfill run {run._id}. Queued progress does not mean the collection + is fully scanned. + +
+ + + + + + + State + + + +
    + {timeline.map(item => ( +
  1. +

    + {item.label} +

    +

    + {formatBackfillTimestamp(item.at)} +

    +
    2. + ))} +
+ + + + + Counters + + + + + + + + Run details + + +
+ + + + + + + + +
+ {run.configId ? ( +

+ + Open config + +

+ ) : null} + {filterJson ? ( + 
+              {filterJson}
+
+ ) : null} + + + {error ? ( + + + Run error + {error} + + ) : null} +
+ ); +} diff --git a/src/components/embeddings/backfills/backfill-filters.tsx b/src/components/embeddings/backfills/backfill-filters.tsx new file mode 100644 index 000000000..a1f7ec879 --- /dev/null +++ b/src/components/embeddings/backfills/backfill-filters.tsx @@ -0,0 +1,131 @@ +'use client'; + +import { usePathname, useRouter, useSearchParams } from 'next/navigation'; +import { Label } from '@/components/ui/label'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { BACKFILL_RUN_STATES } from '@/lib/models/embeddings/backfill'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { + backfillStateLabel, + BackfillListUrlState, +} from '@/lib/models/embeddings/backfill-view'; + +const ALL = 'all'; + +type BackfillFiltersProps = { + query: BackfillListUrlState; + schemas: string[]; + configs: EmbeddingConfig[]; +}; + +function FilterSelect({ + id, + label, + value, + placeholder, + onChange, + options, +}: { + id: string; + label: string; + value: string; + placeholder: string; + onChange: (value: string) => void; + options: { value: string; label: string }[]; +}) { + return ( +
+ + +
+ ); +} + +export function BackfillFilters({ + query, + schemas, + configs, +}: BackfillFiltersProps) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const schemaConfigs = query.schema + ? configs.filter(config => config.schemaName === query.schema) + : configs; + const configOptions = schemaConfigs.map(config => ({ + value: config._id, + label: `${config.schemaName} · ${config.targetField}`, + })); + if ( + query.config && + !configOptions.some(option => option.value === query.config) + ) { + configOptions.unshift({ value: query.config, label: query.config }); + } + + const setFilter = (key: 'schema' | 'config' | 'state', value: string) => { + const params = new URLSearchParams(searchParams.toString()); + if (value === ALL) params.delete(key); + else params.set(key, value); + if (key === 'schema') { + const selected = params.get('config'); + const match = configs.find(config => config._id === selected); + if (match && value !== ALL && match.schemaName !== value) { + params.delete('config'); + } + } + params.delete('skip'); + const next = params.toString(); + router.push(next ? `${pathname}?${next}` : pathname); + }; + + return ( +
+ setFilter('schema', value)} + options={schemas.map(schema => ({ value: schema, label: schema }))} + /> + setFilter('config', value)} + options={configOptions} + /> + setFilter('state', value)} + options={BACKFILL_RUN_STATES.map(state => ({ + value: state, + label: backfillStateLabel(state), + }))} + /> +
+ ); +} diff --git a/src/components/embeddings/backfills/backfill-state-badge.tsx b/src/components/embeddings/backfills/backfill-state-badge.tsx new file mode 100644 index 000000000..ccb2eeda0 --- /dev/null +++ b/src/components/embeddings/backfills/backfill-state-badge.tsx @@ -0,0 +1,49 @@ +'use client'; + +import { + AlertTriangle, + Ban, + CheckCircle2, + Clock, + Play, + type LucideIcon, +} from 'lucide-react'; +import { BackfillRunState } from '@/lib/models/embeddings/backfill'; +import { + backfillStateClass, + backfillStateLabel, +} from '@/lib/models/embeddings/backfill-view'; +import { cn } from '@/lib/utils'; + +const STATE_ICON: Record = { + queued: Clock, + running: Play, + completed: CheckCircle2, + failed: AlertTriangle, + canceled: Ban, +}; + +type BackfillStateBadgeProps = { + state: BackfillRunState; + className?: string; +}; + +export function BackfillStateBadge({ + state, + className, +}: BackfillStateBadgeProps) { + const Icon = STATE_ICON[state]; + const color = backfillStateClass(state); + return ( + + + {backfillStateLabel(state)} + + ); +} diff --git a/src/components/embeddings/backfills/backfills-table.tsx b/src/components/embeddings/backfills/backfills-table.tsx new file mode 100644 index 000000000..5e6782d4f --- /dev/null +++ b/src/components/embeddings/backfills/backfills-table.tsx @@ -0,0 +1,55 @@ +'use client'; + +import Link from 'next/link'; +import { BackfillFilters } from '@/components/embeddings/backfills/backfill-filters'; +import { getBackfillColumns } from '@/components/embeddings/backfills/columns'; +import { useBackfillPolling } from '@/components/embeddings/backfills/use-backfill-polling'; +import { DataTable } from '@/components/ui/data-table'; +import { Button } from '@/components/ui/button'; +import { BackfillRun } from '@/lib/models/embeddings/backfill'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { + BackfillListUrlState, + hasActiveBackfillRuns, +} from '@/lib/models/embeddings/backfill-view'; + +type BackfillsTableProps = { + runs: BackfillRun[]; + count: number; + query: BackfillListUrlState; + configs: EmbeddingConfig[]; + schemas: string[]; + hasConfigs: boolean; +}; + +export function BackfillsTable({ + runs, + count, + query, + configs, + schemas, + hasConfigs, +}: BackfillsTableProps) { + useBackfillPolling(hasActiveBackfillRuns(runs)); + + return ( +
+ + +
+

No backfill runs

+

+ {hasConfigs + ? 'Start a run from a ready config. Progress is of queued documents, not the full collection.' + : 'Create a config first, then start a backfill from this page.'} +

+ {!hasConfigs ? ( + + ) : null} +
+ +
+ ); +} diff --git a/src/components/embeddings/backfills/cancel-backfill-dialog.tsx b/src/components/embeddings/backfills/cancel-backfill-dialog.tsx new file mode 100644 index 000000000..09077ddde --- /dev/null +++ b/src/components/embeddings/backfills/cancel-backfill-dialog.tsx @@ -0,0 +1,49 @@ +'use client'; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; + +type CancelBackfillDialogProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; + pending?: boolean; +}; + +export function CancelBackfillDialog({ + open, + onOpenChange, + onConfirm, + pending = false, +}: CancelBackfillDialogProps) { + return ( + + + + Cancel this run? + + This stops further scheduling. In-flight writes may still finish. + + + + Keep running + + {pending ? 'Canceling…' : 'Cancel run'} + + + + + ); +} diff --git a/src/components/embeddings/backfills/columns.tsx b/src/components/embeddings/backfills/columns.tsx new file mode 100644 index 000000000..5b0075c10 --- /dev/null +++ b/src/components/embeddings/backfills/columns.tsx @@ -0,0 +1,82 @@ +'use client'; + +import { ColumnDef } from '@tanstack/react-table'; +import Link from 'next/link'; +import { BackfillStateBadge } from '@/components/embeddings/backfills/backfill-state-badge'; +import { BackfillRun } from '@/lib/models/embeddings/backfill'; +import { + backfillProgress, + formatBackfillTimestamp, +} from '@/lib/models/embeddings/backfill-view'; + +export function getBackfillColumns(): ColumnDef[] { + return [ + { + id: 'schema', + header: 'Schema', + cell: ({ row }) => ( + + {row.original.schemaName} + + ), + }, + { + id: 'state', + header: 'State', + cell: ({ row }) => , + }, + { + id: 'progress', + header: 'Queued progress', + cell: ({ row }) => { + const progress = backfillProgress(row.original); + return ( +
+

+ {progress.queued > 0 + ? `${progress.done.toLocaleString()} / ${progress.queued.toLocaleString()}` + : progress.scanning + ? 'Scanning' + : '—'} +

+

+ {progress.scanned.toLocaleString()} scanned + {progress.failed > 0 + ? ` · ${progress.failed.toLocaleString()} failed` + : ''} +

+
+ ); + }, + }, + { + id: 'config', + header: 'Config', + cell: ({ row }) => + row.original.configId ? ( + + {row.original.configId} + + ) : ( + All enabled + ), + }, + { + id: 'updated', + header: 'Updated', + cell: ({ row }) => ( + + {formatBackfillTimestamp( + row.original.updatedAt ?? row.original.createdAt + )} + + ), + }, + ]; +} diff --git a/src/components/embeddings/backfills/start-backfill-dialog.tsx b/src/components/embeddings/backfills/start-backfill-dialog.tsx new file mode 100644 index 000000000..1109d7ff8 --- /dev/null +++ b/src/components/embeddings/backfills/start-backfill-dialog.tsx @@ -0,0 +1,289 @@ +'use client'; + +import { useMemo, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { ChevronDown } from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from '@/components/ui/collapsible'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from '@/components/ui/dialog'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Switch } from '@/components/ui/switch'; +import { Textarea } from '@/components/ui/textarea'; +import { startBackfill } from '@/lib/api/embeddings'; +import { toast } from '@/lib/hooks/use-toast'; +import { EmbeddingConfig } from '@/lib/models/embeddings/config'; +import { formatEmbeddingsApiError } from '@/lib/models/embeddings/errors'; +import { + defaultBackfillBatchSize, + maxAllowedBatchSize, + parseBatchSizeInput, + parseOperatorFilterJson, +} from '@/lib/models/embeddings/backfill-view'; + +const ALL_CONFIGS = 'all'; + +type StartBackfillDialogProps = { + configs: EmbeddingConfig[]; + schemas: string[]; + maxBatchSize?: number; + defaultSchema?: string; + defaultConfig?: string; +}; + +export function StartBackfillDialog({ + configs, + schemas, + maxBatchSize, + defaultSchema, + defaultConfig, +}: StartBackfillDialogProps) { + const router = useRouter(); + const allowedMax = maxAllowedBatchSize(maxBatchSize); + const [open, setOpen] = useState(false); + const [schemaName, setSchemaName] = useState(defaultSchema ?? ''); + const [configId, setConfigId] = useState(defaultConfig ?? ALL_CONFIGS); + const [onlyMissing, setOnlyMissing] = useState(true); + const [batchSize, setBatchSize] = useState( + String(defaultBackfillBatchSize(maxBatchSize)) + ); + const [filterText, setFilterText] = useState(''); + const [filterError, setFilterError] = useState(); + const [batchError, setBatchError] = useState(); + const [pending, setPending] = useState(false); + + const schemaConfigs = useMemo( + () => + schemaName + ? configs.filter(config => config.schemaName === schemaName) + : configs, + [configs, schemaName] + ); + + const reset = () => { + const matched = configs.find(config => config._id === defaultConfig); + setSchemaName(defaultSchema ?? matched?.schemaName ?? ''); + setConfigId(defaultConfig ?? ALL_CONFIGS); + setOnlyMissing(true); + setBatchSize(String(defaultBackfillBatchSize(maxBatchSize))); + setFilterText(''); + setFilterError(undefined); + setBatchError(undefined); + }; + + const submit = async () => { + const batch = parseBatchSizeInput(batchSize, maxBatchSize); + const filter = parseOperatorFilterJson(filterText); + setBatchError(batch.ok ? undefined : batch.error); + setFilterError(filter.ok ? undefined : filter.error); + if (!schemaName) return; + if (!batch.ok || !filter.ok) return; + setPending(true); + try { + const result = await startBackfill({ + schemaName, + batchSize: batch.batchSize, + onlyMissing, + configId: configId === ALL_CONFIGS ? undefined : configId, + filter: filter.filter, + }); + toast({ + title: 'Backfill queued', + description: + result.warnings.length > 0 + ? result.warnings.join(' ') + : `${result.queued.toLocaleString()} run${result.queued === 1 ? '' : 's'} queued`, + }); + setOpen(false); + reset(); + router.refresh(); + } catch (error) { + toast({ + title: 'Could not start backfill', + description: formatEmbeddingsApiError(error), + variant: 'destructive', + }); + } finally { + setPending(false); + } + }; + + const disabled = configs.length === 0; + + return ( + { + setOpen(next); + if (next) reset(); + }} + > + + + + +
{ + event.preventDefault(); + void submit(); + }} + className="grid gap-4" + > + + Start backfill + + Queue embedding generation for documents in a schema. Progress is + of queued documents, not the full collection. + + +
+ + +
+
+ + +
+
+ + { + setBatchSize(event.target.value); + setBatchError(undefined); + }} + /> +

+ Default 100. Maximum {allowedMax.toLocaleString()}. +

+ {batchError ? ( +

{batchError}

+ ) : null} +
+ + + + Advanced filter + + + + +