Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 14 additions & 23 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 12 additions & 3 deletions packages/compass-assistant/test/tool-calls.eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/

import { strict as assert } from 'node:assert';
import { MessageChannel } from 'node:worker_threads';
import { createOpenAI } from '@ai-sdk/openai';
import { streamText } from 'ai';
import type { ToolSet, ToolCallPart } from 'ai';
Expand All @@ -17,6 +18,7 @@ import {
ToolsController,
type ToolGroup,
} from '@mongodb-js/compass-generative-ai/provider';
import { ToolsHost } from '@mongodb-js/compass-generative-ai/utility';
import { createNoopLogger } from '@mongodb-js/compass-logging/provider';
import { createNoopTrack } from '@mongodb-js/compass-telemetry/provider';
import { defaultPreferencesInstance } from '@mongodb-js/compass-preferences-model';
Expand Down Expand Up @@ -65,13 +67,20 @@ const DEFAULT_EVAL_TASK_CONFIG: EvalTaskConfig = {

const logger = createNoopLogger('EVAL-TOOLS-CONTROLLER');

// There is no utility process in Node, so the host runs in this one, on the
// other end of a plain MessageChannel. Held in a variable so the run can
// release its server and driver connection when it finishes.
const evalChannel = new MessageChannel();
const evalHost = new ToolsHost(evalChannel.port2, { logger });

const toolsController = new ToolsController({
logger,
getTelemetryAnonymousId: () => 'eval-anonymous-id',
track: createNoopTrack(),
enableMCPTelemetry: false,
preferences: defaultPreferencesInstance,
atlasAdminApi: {} as AtlasAdminApiService,
openPort: () => evalChannel.port1,
});

let toolsInitialized = false;
Expand All @@ -83,9 +92,6 @@ let toolsInitialized = false;
async function ensureToolsInitialized(): Promise<void> {
if (toolsInitialized) return;
await toolsController.startServer();
if (!toolsController.server) {
throw new Error('Failed to start ToolsController server for eval tools');
}
toolsInitialized = true;
}

Expand Down Expand Up @@ -218,6 +224,8 @@ function createToolCallAssistantTask(config: EvalTaskConfig) {
connectionConfig,
});

assert(tools['find'], 'ToolsController did not provide database tools');

const contextPrompt = buildContextPromptText({
custom,
connectionString: connectionConfig.connectionString,
Expand Down Expand Up @@ -277,6 +285,7 @@ async function main() {
});
} finally {
await toolsController.stopServer();
await evalHost.stop();
await client.close();
await cluster.close();
// FIXME(COMPASS-10486): We need to have this otherwise the process hangs.
Expand Down
6 changes: 3 additions & 3 deletions packages/compass-e2e-tests/tests/assistant.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,12 @@ describe('MongoDB Assistant (with real backend)', function () {
{ timeout: 30_000 }
);

// Expand the tool call card to see the response
const expandButton = chatMessages.$(
'button[aria-label="Expand additional content"]'
);
await expandButton.waitForDisplayed();
await browser.clickVisible(expandButton);
if (await expandButton.isExisting()) {
await browser.clickVisible(expandButton);
}

// Wait for the expanded content to include the response with structuredContent.databases
await browser.waitUntil(
Expand Down
10 changes: 6 additions & 4 deletions packages/compass-generative-ai/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@
"compass:main": "src/index.ts",
"exports": {
".": "./dist/index.js",
"./provider": "./dist/provider.js"
"./provider": "./dist/provider.js",
"./utility": "./dist/tools/infrastructure/tools-host.js"
},
"compass:exports": {
".": "./src/index.ts",
"./provider": "./src/provider.tsx"
"./provider": "./src/provider.tsx",
"./utility": "./src/tools/infrastructure/tools-host.ts"
},
"types": "./dist/index.d.ts",
"scripts": {
Expand Down Expand Up @@ -70,7 +72,8 @@
"@mongodb-js/shell-bson-parser": "^1.6.3",
"@mongosh/service-provider-core": "^5.2.0",
"@mongosh/service-provider-node-driver": "^5.2.0",
"ai": "^7.0.0",
"ai": "^6.0.3",
"bson": "^7.3.2",
"ipaddr.js": "^2.5.0",
"mongodb": "^7.5.0",
"mongodb-connection-string-url": "^7.0.2",
Expand All @@ -96,7 +99,6 @@
"@types/sinon-chai": "^3.2.5",
"autoevals": "^0.4.0",
"braintrust": "^3.35.0",
"bson": "^7.3.2",
"chai": "^4.3.6",
"depcheck": "^1.4.1",
"electron-mocha": "^12.2.0",
Expand Down
22 changes: 20 additions & 2 deletions packages/compass-generative-ai/src/provider.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
import { atlasAdminApiServiceLocator } from '@mongodb-js/atlas-admin-api/provider';
import { telemetryLocator } from '@mongodb-js/compass-telemetry/provider';
import { ToolsController } from './tools-controller';
import type { ToolsRpcPort } from './tools/infrastructure/messages';
import { AtlasAiService } from './atlas-ai-service';

const AtlasAiServiceContext = createContext<AtlasAiService | null>(null);
Expand Down Expand Up @@ -60,8 +61,17 @@ export { AtlasAiService } from './atlas-ai-service';
const ToolsControllerContext = createContext<ToolsController | null>(null);

export const ToolsControllerProvider: React.FC<{
/**
* How the controller reaches its tools host. Defaults to the desktop
* utility process; the browser has none, so compass-web provides a loopback
* that runs the same host in the page.
*/
openPort?: () => ToolsRpcPort;
children?: React.ReactNode;
}> = createServiceProvider(function ToolsControllerProvider({ children }) {
}> = createServiceProvider(function ToolsControllerProvider({
openPort,
children,
}) {
const logger = useLogger('TOOLS-CONTROLLER');
const preferences = preferencesLocator();
const atlasAdminApi = atlasAdminApiServiceLocator();
Expand All @@ -78,8 +88,16 @@ export const ToolsControllerProvider: React.FC<{
enableMCPTelemetry: false,
preferences,
atlasAdminApi,
openPort,
});
}, [logger, telemetryAnonymousId, track, preferences, atlasAdminApi]);
}, [
logger,
telemetryAnonymousId,
track,
preferences,
atlasAdminApi,
openPort,
]);

useEffect(() => {
return () => {
Expand Down
Loading
Loading