diff --git a/.eslintignore b/.eslintignore index 02340bb2f..2c2265615 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,3 +2,4 @@ node_modules out dist scripts +src/vscode-dts diff --git a/.eslintrc.js b/.eslintrc.js index 8324538d6..24170e282 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -63,6 +63,12 @@ module.exports = { 'error', { prefer: 'type-imports' }, ], + '@typescript-eslint/explicit-function-return-type': [ + 'warn', + { + allowHigherOrderFunctions: true, + }, + ], }, parserOptions: { project: ['./tsconfig.json'], // Specify it only for TypeScript files. diff --git a/.github/workflows/actions/test-and-build/action.yaml b/.github/workflows/actions/test-and-build/action.yaml index 99dd9eda0..ce6d4bc08 100644 --- a/.github/workflows/actions/test-and-build/action.yaml +++ b/.github/workflows/actions/test-and-build/action.yaml @@ -70,6 +70,8 @@ runs: shell: bash - name: Run Tests + env: + NODE_OPTIONS: "--max_old_space_size=4096" run: | npm run test shell: bash diff --git a/.gitignore b/.gitignore index e41f34cfa..e5785ab14 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ constants.json .env .eslintcache .sbom +src/test/ai-accuracy-tests/test-results.html diff --git a/.vscode/launch.json b/.vscode/launch.json index 2ca2ae136..e7f6ca212 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,7 +14,7 @@ "--extensionDevelopmentPath=${workspaceFolder}" ], "outFiles": [ - "${workspaceFolder}/out/**/*.js" + "${workspaceFolder}/dist/**/*.js" ], "preLaunchTask": "${defaultBuildTask}" }, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ebf5280cb..0e3168a2c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,12 @@ npm run watch 2. Inside of [VS Code Insiders](https://code.visualstudio.com/insiders/) open this directory and press `F5` to begin debugging the extension. This should launch a new VSCode window which is running the extension. +### Using Proposed API + +The vscode extension will occasionally need to use [proposed API](https://code.visualstudio.com/api/advanced-topics/using-proposed-api) that haven't been promoted to stable yet. To enable an API proposal, add it to the `enabledApiProposals` section in `package.json`, then run `cd src/vscode-dts && npx @vscode/dts dev` to install the type definitions for the API you want to enable. + +**Note**: Using proposed API is only possible during local development and will prevent publishing the extension. + #### Code Tour - `out` Compiled extension code diff --git a/README.md b/README.md index 54ec00cc1..ecad90e60 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,25 @@ Connect to Atlas Stream Processing instances and develop stream processors using ![Atlas Stream Processing Playground](resources/screenshots/atlas-stream-processing.png) +### MongoDB Copilot Participant + +Use natural language to interact with your clusters and generate MongoDB-related code with GitHub Copilot Chat in VS Code. + +_Note: To use the MongoDB Participant, you must have the [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot) extension. By using Copilot Chat you agree to [GitHub Copilot chat preview terms](https://docs.github.com/en/copilot/responsible-use-of-github-copilot-features/responsible-use-of-github-copilot-chat-in-your-ide). Find more details about the MongoDB GenAI Features in the [FAQ](https://www.mongodb.com/docs/generative-ai-faq/)._ + +#### How to use the MongoDB Participant + +1. Enter `@MongoDB` in the chat input field to start a conversation with the MongoDB Participant. + +![MongoDB Participant](resources/screenshots/mongodb-participant.png) + +2. Start typing `/` in the chat window to get the list of available chat participant commands. + +![MongoDB Participant Commands](resources/screenshots/mongodb-participant-commands.png) + +- `/docs` is a participant command that finds answers to coding-related questions in the [MongoDB documentation](https://www.mongodb.com/docs/). +- `/query` is a participant command that generates MongoDB queries from natural language to be used with a connected MongoDB cluster. It generates both queries and aggregations depending on the complexity of the request. It utilizes schema to reduce model hallucinations. It provides a code action to open generated code in a playground and an action to directly run the code from the Copilot chat interface. +- `/schema` is a participant command that analyzes and returns information about a collection's schema. ## Extension Settings @@ -88,6 +107,8 @@ Connect to Atlas Stream Processing instances and develop stream processors using | `mdb.showMongoDBHelpExplorer` | Show or hide the MongoDB Help panel. | `true` | | `mdb.defaultLimit` | The number of documents to fetch when viewing documents from a collection. | `10` | | `mdb.confirmRunAll` | Show a confirmation message before running commands in a playground. | `true` | +| `mdb.confirmRunCopilotCode` | Show a confirmation message before running code generated by the MongoDB participant. | `true` | +| `mdb.useSampleDocsInCopilot` | Enable sending sample field values with the VSCode copilot chat @MongoDB participant /query command. | `false` | | `mdb.confirmDeleteDocument` | Show a confirmation message before deleting a document in the tree view. | `true` | | `mdb.persistOIDCTokens` | Remain logged in when using the MONGODB-OIDC authentication mechanism for MongoDB server connection. Access tokens are encrypted using the system keychain before being stored. | `true` | | `mdb.showOIDCDeviceAuthFlow` | Opt-in and opt-out for diagnostic and telemetry collection. | `true` | @@ -97,6 +118,8 @@ Connect to Atlas Stream Processing instances and develop stream processors using | `mdb.useDefaultTemplateForPlayground` | Choose whether to use the default template for playground files or to start with an empty playground editor. | `true` | | `mdb.uniqueObjectIdPerCursor` | The default behavior is to generate a single ObjectId and insert it on all cursors. Set to true to generate a unique ObjectId per cursor instead. | `false` | | `mdb.sendTelemetry` | Opt-in and opt-out for diagnostic and telemetry collection. | `true` | +| `mdb.confirmRunCopilotCode` | Show a confirmation message before running code generated by the MongoDB participant. | `true` | +| `mdb.useSampleDocsInCopilot` | Enable sending sample field values with the VSCode copilot chat @MongoDB participant /query command. | `false` | ## Additional Settings diff --git a/package-lock.json b/package-lock.json index b94d8bcc8..14155662c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,7 +60,6 @@ "@types/chai": "^4.3.17", "@types/debug": "^4.1.12", "@types/glob": "^7.2.0", - "@types/jest": "^26.0.24", "@types/micromatch": "^4.0.9", "@types/mkdirp": "^2.0.0", "@types/mocha": "^8.2.3", @@ -74,7 +73,7 @@ "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", "@vscode/test-electron": "^2.4.1", - "@vscode/vsce": "^2.31.1", + "@vscode/vsce": "^3.1.0", "buffer": "^6.0.3", "chai": "^4.5.0", "chai-as-promised": "^7.1.2", @@ -94,15 +93,18 @@ "mocha-junit-reporter": "^2.2.1", "mocha-multi": "^1.1.7", "mongodb-client-encryption": "^6.0.1", + "mongodb-rag-core": "^0.4.1", "mongodb-runner": "^5.6.4", "node-fetch": "^2.7.0", "node-loader": "^0.6.0", "npm-run-all": "^4.1.5", + "openai": "^4.55.7", "ora": "^5.4.1", "path-browserify": "^1.0.1", "pre-commit": "^1.2.2", "prettier": "^2.8.8", "process": "^0.11.10", + "rewiremock": "^3.14.5", "sinon": "^9.2.4", "sinon-chai": "^3.7.0", "source-map-support": "^0.5.21", @@ -144,6 +146,30 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.25.2", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.25.2.tgz", + "integrity": "sha512-F1Hck/asswwidFLtGdMg3XYgRxEUfygNbpkq5KEaEGsHNaSfxeX18/uZGQCL0oQNcj/tYNx8BaFXVwRhFDi45g==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { + "version": "18.19.50", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.50.tgz", + "integrity": "sha512-xonK+NRrMBRtkL1hVCc3G+uXtjh1Al4opBLjqVmipe5ZAaBYWW6cNAiBVZ1BvmkBhep698rP3UM3aRAdSALuhg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@asamuzakjp/dom-selector": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-2.0.2.tgz", @@ -855,6 +881,35 @@ } } }, + "node_modules/@azure-rest/core-client": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-1.4.0.tgz", + "integrity": "sha512-ozTDPBVUDR5eOnMIwhggbnVmOrka4fXCs8n8mvUo4WLLc38kki6bAOByDoVZZPz/pZy2jMt2kwfpvy/UjALj6w==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-rest-pipeline": "^1.5.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure-rest/core-client/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/abort-controller": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", @@ -967,6 +1022,18 @@ "node": ">= 14" } }, + "node_modules/@azure/core-sse": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@azure/core-sse/-/core-sse-2.1.3.tgz", + "integrity": "sha512-KSSdIKy8kvWCpYr8Hzpu22j3wcXsVTYE0IlgmI1T/aHvBDsLgV91y90UTfVWnuiuApRLCCVC4gS09ApBGOmYQA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/core-tracing": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", @@ -1102,6 +1169,24 @@ "node": ">=16" } }, + "node_modules/@azure/openai": { + "version": "1.0.0-beta.12", + "resolved": "https://registry.npmjs.org/@azure/openai/-/openai-1.0.0-beta.12.tgz", + "integrity": "sha512-qKblxr6oVa8GsyNzY+/Ub9VmEsPYKhBrUrPaNEQiM+qrxnBPVm9kaeqGFFb/U78Q2zOabmhF9ctYt3xBW0nWnQ==", + "dev": true, + "dependencies": { + "@azure-rest/core-client": "^1.1.7", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.13.0", + "@azure/core-sse": "^2.0.0", + "@azure/core-util": "^1.4.0", + "@azure/logger": "^1.0.3", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", @@ -1755,6 +1840,15 @@ "w3c-keyname": "^2.2.4" } }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "dev": true, + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1777,6 +1871,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dev": true, + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@discoveryjs/json-ext": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", @@ -2209,22 +2314,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/@jest/types": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.6.2.tgz", - "integrity": "sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^15.0.0", - "chalk": "^4.0.0" - }, - "engines": { - "node": ">= 10.14.2" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -2278,6 +2367,130 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@langchain/anthropic": { + "version": "0.2.18", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.2.18.tgz", + "integrity": "sha512-4ZDTxMwGKTPRAi2Supu/faBSmwPIm/ga5QlazyO78Mf/8QbDR2DcvM5394FAy+X/nRAfnMbyXteO5IRJm653gw==", + "dev": true, + "dependencies": { + "@anthropic-ai/sdk": "^0.25.2", + "@langchain/core": ">=0.2.21 <0.3.0", + "fast-xml-parser": "^4.4.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core": { + "version": "0.2.32", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.32.tgz", + "integrity": "sha512-S27M+9Qou2qtcLfFGEvANkJ/zHq5XApeQsR6Q4I7C6v9x07eoYr558h6vVy6WQmKcksgbCIJ854ikwp173wBjA==", + "dev": true, + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.1.43", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core/node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true + }, + "node_modules/@langchain/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@langchain/core/node_modules/langsmith": { + "version": "0.1.55", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.55.tgz", + "integrity": "sha512-6NVtI04UUnIY59I/imOX02FG/QMGfqStu8tiJtyyreKMv2GAN0EE9Z5Ap1wzOe6v8ukEcV3NwEO2LYOPwup1PQ==", + "dev": true, + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/@langchain/core/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@langchain/openai": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.2.10.tgz", + "integrity": "sha512-ph5sYDAmhP55Fs3TW3+LXiqF+r/5zaaNO2tur9p2Otr8KWNDSgp5ezfPki1WWfuUJVoSQ+6HDYtr6n2V5N1Lew==", + "dev": true, + "dependencies": { + "@langchain/core": ">=0.2.26 <0.3.0", + "js-tiktoken": "^1.0.12", + "openai": "^4.57.3", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@leafygreen-ui/a11y": { "version": "1.4.13", "resolved": "https://registry.npmjs.org/@leafygreen-ui/a11y/-/a11y-1.4.13.tgz", @@ -5706,40 +5919,6 @@ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", - "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", - "dev": true - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", - "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz", - "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==", - "dev": true, - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/jest": { - "version": "26.0.24", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.24.tgz", - "integrity": "sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w==", - "dev": true, - "dependencies": { - "jest-diff": "^26.0.0", - "pretty-format": "^26.0.0" - } - }, "node_modules/@types/json-schema": { "version": "7.0.12", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", @@ -5796,6 +5975,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -5806,6 +5995,12 @@ "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" }, + "node_modules/@types/qs": { + "version": "6.9.15", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", + "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "dev": true + }, "node_modules/@types/react": { "version": "17.0.80", "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.80.tgz", @@ -5856,6 +6051,12 @@ "@types/node": "*" } }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true + }, "node_modules/@types/scheduler": { "version": "0.16.3", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", @@ -5910,6 +6111,12 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "dev": true + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", @@ -5941,21 +6148,6 @@ "@types/webidl-conversions": "*" } }, - "node_modules/@types/yargs": { - "version": "15.0.15", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.15.tgz", - "integrity": "sha512-IziEYMU9XoVj8hWg7k+UJrXALkGFjWJhn5QFEv9q4p+v40oZhSuC135M38st8XPjICL7Ey4TV64ferBGUoJhBg==", - "dev": true, - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.0", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", - "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", - "dev": true - }, "node_modules/@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -6333,10 +6525,11 @@ } }, "node_modules/@vscode/vsce": { - "version": "2.31.1", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.31.1.tgz", - "integrity": "sha512-LwEQFKXV21C4/brvGPH/9+7ZOUM5cbK7oJ4fVmy0YG75NIy1HV8eMSoBZrl+u23NxpAhor62Cu1aI+JFtCtjSg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.1.0.tgz", + "integrity": "sha512-fwdfp1Ol+bZtlSGkpcd/nztfo6+SVsTOMWjZ/+a88lVtUn7gXNbSu7dbniecl5mz4vINl+oaVDVtVdGbJDApmw==", "dev": true, + "license": "MIT", "dependencies": { "@azure/identity": "^4.1.0", "@vscode/vsce-sign": "^2.0.0", @@ -6350,7 +6543,7 @@ "hosted-git-info": "^4.0.2", "jsonc-parser": "^3.2.0", "leven": "^3.1.0", - "markdown-it": "^12.3.2", + "markdown-it": "^14.1.0", "mime": "^1.3.4", "minimatch": "^3.0.3", "parse-semver": "^1.1.1", @@ -6367,7 +6560,7 @@ "vsce": "vsce" }, "engines": { - "node": ">= 16" + "node": ">= 20" }, "optionalDependencies": { "keytar": "^7.7.0" @@ -6876,6 +7069,18 @@ "node": ">= 14" } }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dev": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -7119,6 +7324,31 @@ "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true }, + "node_modules/assert": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.1.tgz", + "integrity": "sha512-zzw1uCAgLbsKwBfFc8CX78DDg+xZeBksSO3vwVIDDN5i94eOrPsSSyiVhmsSABFDM/OcpE2aagCat9dnWQLG1A==", + "dev": true, + "dependencies": { + "object.assign": "^4.1.4", + "util": "^0.10.4" + } + }, + "node_modules/assert/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true + }, + "node_modules/assert/node_modules/util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "dev": true, + "dependencies": { + "inherits": "2.0.3" + } + }, "node_modules/assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -7149,6 +7379,12 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -7171,6 +7407,17 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/azure-devops-node-api": { "version": "12.5.0", "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-12.5.0.tgz", @@ -7225,6 +7472,22 @@ "npm": ">=6" } }, + "node_modules/babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==", + "dev": true, + "dependencies": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, + "node_modules/babel-runtime/node_modules/regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -7549,6 +7812,15 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "dev": true, + "dependencies": { + "pako": "~1.0.5" + } + }, "node_modules/browserslist": { "version": "4.23.3", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.3.tgz", @@ -7711,6 +7983,12 @@ "node": ">=10.0.0" } }, + "node_modules/builtin-status-codes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", + "dev": true + }, "node_modules/bundle-name": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz", @@ -7759,13 +8037,18 @@ } }, "node_modules/call-bind": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", - "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.1", - "set-function-length": "^1.1.1" + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7788,6 +8071,18 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001651", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz", @@ -8095,6 +8390,16 @@ "node": ">=16" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -8111,6 +8416,31 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -8126,6 +8456,16 @@ "node": ">=0.1.90" } }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dev": true, + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -8147,6 +8487,21 @@ "node": ">=14" } }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/compare-module-exports": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/compare-module-exports/-/compare-module-exports-2.1.0.tgz", + "integrity": "sha512-3Lc0sTIuX1jmY2K2RrXRJOND6KsRTX2D4v3+eu1PDptsuJZVK4LZc852eZa9I+avj0NrUKlTNgqvccNOH6mbGg==", + "dev": true + }, "node_modules/compass-preferences-model": { "version": "2.27.0", "resolved": "https://registry.npmjs.org/compass-preferences-model/-/compass-preferences-model-2.27.0.tgz", @@ -8256,6 +8611,18 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/console-browserify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", + "dev": true + }, + "node_modules/constants-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==", + "dev": true + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -8293,6 +8660,14 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, + "node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", + "dev": true, + "hasInstallScript": true + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -8941,16 +9316,19 @@ } }, "node_modules/define-data-property": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", - "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", "dependencies": { - "get-intrinsic": "^1.2.1", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.0" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/define-lazy-prop": { @@ -9053,18 +9431,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/depcheck/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/depcheck/node_modules/minimatch": { "version": "7.4.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.6.tgz", @@ -9173,15 +9539,6 @@ "node": ">=0.3.1" } }, - "node_modules/diff-sequences": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-26.6.2.tgz", - "integrity": "sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q==", - "dev": true, - "engines": { - "node": ">= 10.14.2" - } - }, "node_modules/diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -9251,6 +9608,16 @@ "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" } }, + "node_modules/domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true, + "engines": { + "node": ">=0.4", + "npm": ">=1.2" + } + }, "node_modules/domelementtype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", @@ -9559,6 +9926,12 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "devOptional": true }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "dev": true + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -9686,6 +10059,25 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-get-iterator": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", @@ -10710,6 +11102,71 @@ "safe-buffer": "^5.1.1" } }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/execa/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -10731,6 +11188,12 @@ "node": ">=0.10.0" } }, + "node_modules/exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==", + "dev": true + }, "node_modules/express": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", @@ -10785,6 +11248,18 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -10930,6 +11405,12 @@ "pend": "~1.2.0" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "dev": true + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -11103,6 +11584,12 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "dev": true + }, "node_modules/focus-trap": { "version": "6.9.4", "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-6.9.4.tgz", @@ -11125,6 +11612,26 @@ "react-dom": ">=16.3.0" } }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -11282,6 +11789,34 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "dev": true + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dev": true, + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -11310,6 +11845,15 @@ "node": ">= 0.6" } }, + "node_modules/front-matter": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/front-matter/-/front-matter-4.0.2.tgz", + "integrity": "sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==", + "dev": true, + "dependencies": { + "js-yaml": "^3.13.1" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -11453,15 +11997,19 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", - "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", "dependencies": { + "es-errors": "^1.3.0", "function-bind": "^1.1.2", "has-proto": "^1.0.1", "has-symbols": "^1.0.3", "hasown": "^2.0.0" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -11748,6 +12296,21 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "dev": true, + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -11880,11 +12443,11 @@ } }, "node_modules/has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", "dependencies": { - "get-intrinsic": "^1.1.1" + "es-define-property": "^1.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -12180,6 +12743,12 @@ "node": ">=10.19.0" } }, + "node_modules/https-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==", + "dev": true + }, "node_modules/https-proxy-agent": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", @@ -12201,6 +12770,15 @@ "node": ">=14.18.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/husky": { "version": "9.1.4", "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.4.tgz", @@ -12734,6 +13312,15 @@ "resolved": "https://registry.npmjs.org/is-electron-renderer/-/is-electron-renderer-2.0.1.tgz", "integrity": "sha512-pRlQnpaCFhDVPtkXkP+g9Ybv/CjbiQDjnKFQTEjpBfDKeV6dRDBczuFRDpM6DVfk2EjpMS8t5kwE5jPnqYl3zA==" }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -13080,30 +13667,6 @@ "resolved": "https://registry.npmjs.org/javascript-stringify/-/javascript-stringify-2.1.0.tgz", "integrity": "sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg==" }, - "node_modules/jest-diff": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-26.6.2.tgz", - "integrity": "sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^26.6.2", - "jest-get-type": "^26.3.0", - "pretty-format": "^26.6.2" - }, - "engines": { - "node": ">= 10.14.2" - } - }, - "node_modules/jest-get-type": { - "version": "26.3.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-26.3.0.tgz", - "integrity": "sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==", - "dev": true, - "engines": { - "node": ">= 10.14.2" - } - }, "node_modules/jose": { "version": "4.15.7", "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.7.tgz", @@ -13112,6 +13675,15 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-tiktoken": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.14.tgz", + "integrity": "sha512-Pk3l3WOgM9joguZY2k52+jH82RtABRgB5RdGFZNUGbOKGMVlNmafcPA3b0ITcCZPu1L9UclP1tne6aw7ZI4Myg==", + "dev": true, + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -13452,6 +14024,12 @@ "node": ">=0.10.0" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "dev": true + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -13489,12 +14067,13 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, "node_modules/linkify-it": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", - "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", "dev": true, + "license": "MIT", "dependencies": { - "uc.micro": "^1.0.1" + "uc.micro": "^2.0.0" } }, "node_modules/load-json-file": { @@ -13569,6 +14148,12 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash._reinterpolate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", + "integrity": "sha512-xYHt68QRoYGjeeM/XOE1uJtvXQAgvszfBhjV4yvsQH0u2i9I6cI6c6/eG4Hh3UAOVn0y/xAXwmTzEay49Q//HA==", + "dev": true + }, "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -13628,6 +14213,31 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "dev": true }, + "node_modules/lodash.some": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz", + "integrity": "sha512-j7MJE+TuT51q9ggt4fSgVqro163BEFjAt3u97IqU+JA2DkWl80nFTrowzLpZ/BnpN7rrl0JA/593NAdd8p/scQ==", + "dev": true + }, + "node_modules/lodash.template": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", + "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==", + "dev": true, + "dependencies": { + "lodash._reinterpolate": "^3.0.0", + "lodash.templatesettings": "^4.0.0" + } + }, + "node_modules/lodash.templatesettings": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz", + "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==", + "dev": true, + "dependencies": { + "lodash._reinterpolate": "^3.0.0" + } + }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -13644,6 +14254,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/logform": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz", + "integrity": "sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==", + "dev": true, + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -13743,35 +14370,29 @@ "dev": true }, "node_modules/markdown-it": { - "version": "12.3.2", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", - "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^2.0.1", - "entities": "~2.1.0", - "linkify-it": "^3.0.1", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" }, "bin": { - "markdown-it": "bin/markdown-it.js" + "markdown-it": "bin/markdown-it.mjs" } }, "node_modules/markdown-it/node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "node_modules/markdown-it/node_modules/entities": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", - "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", "dev": true, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } + "license": "Python-2.0" }, "node_modules/matcher": { "version": "3.0.0", @@ -13816,10 +14437,11 @@ "license": "CC0-1.0" }, "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", - "dev": true + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "dev": true, + "license": "MIT" }, "node_modules/media-typer": { "version": "0.3.0", @@ -14511,6 +15133,65 @@ "lodash": "^4.17.21" } }, + "node_modules/mongodb-rag-core": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/mongodb-rag-core/-/mongodb-rag-core-0.4.1.tgz", + "integrity": "sha512-McgQyKeex2e2qR8PNo4zmn5XEAnPs5jmdtRZn5f9GAY1tQGjN+JS7ePYgLKwsEDUxRsR56rrtn9ExJgdFo9gFg==", + "dev": true, + "dependencies": { + "@azure/openai": "^1.0.0-beta.5", + "@langchain/anthropic": "^0.2.15", + "@langchain/core": "^0.2.27", + "@langchain/openai": "^0.2.7", + "common-tags": "^1", + "dotenv": "^16.3.1", + "exponential-backoff": "^3.1.1", + "front-matter": "^4.0.2", + "gray-matter": "^4.0.3", + "mongodb": "^6.3.0", + "openai": "^3", + "toml": "^3.0.0", + "typechat": "^0.0.10", + "winston": "^3", + "yaml": "^2.3.1", + "zod": "^3.21.4" + }, + "engines": { + "node": ">=18", + "npm": ">=8" + } + }, + "node_modules/mongodb-rag-core/node_modules/axios": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.14.8" + } + }, + "node_modules/mongodb-rag-core/node_modules/openai": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", + "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", + "dev": true, + "dependencies": { + "axios": "^0.26.0", + "form-data": "^4.0.0" + } + }, + "node_modules/mongodb-rag-core/node_modules/yaml": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.1.tgz", + "integrity": "sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/mongodb-redact": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/mongodb-redact/-/mongodb-redact-1.1.3.tgz", @@ -14707,6 +15388,15 @@ "object-assign": "^4.1.0" } }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -14869,6 +15559,115 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/node-libs-browser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", + "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "dev": true, + "dependencies": { + "assert": "^1.1.1", + "browserify-zlib": "^0.2.0", + "buffer": "^4.3.0", + "console-browserify": "^1.1.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.11.0", + "domain-browser": "^1.1.1", + "events": "^3.0.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "0.0.1", + "process": "^0.11.10", + "punycode": "^1.2.4", + "querystring-es3": "^0.2.0", + "readable-stream": "^2.3.3", + "stream-browserify": "^2.0.1", + "stream-http": "^2.7.2", + "string_decoder": "^1.0.0", + "timers-browserify": "^2.0.4", + "tty-browserify": "0.0.0", + "url": "^0.11.0", + "util": "^0.11.0", + "vm-browserify": "^1.0.1" + } + }, + "node_modules/node-libs-browser/node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dev": true, + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/node-libs-browser/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/node-libs-browser/node_modules/path-browserify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz", + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==", + "dev": true + }, + "node_modules/node-libs-browser/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", + "dev": true + }, + "node_modules/node-libs-browser/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/node-libs-browser/node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/node-libs-browser/node_modules/readable-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/node-libs-browser/node_modules/stream-browserify": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "dev": true, + "dependencies": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + } + }, + "node_modules/node-libs-browser/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/node-loader": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/node-loader/-/node-loader-0.6.0.tgz", @@ -15090,6 +15889,18 @@ "which": "bin/which" } }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/nth-check": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", @@ -15127,9 +15938,12 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -15262,6 +16076,15 @@ "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dev": true, + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", @@ -15293,6 +16116,43 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.59.0.tgz", + "integrity": "sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "@types/qs": "^6.9.15", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "qs": "^6.10.3" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.48", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.48.tgz", + "integrity": "sha512-7WevbG4ekUcRQSZzOwxWgi5dZmTak7FaxXDoW7xVxPBmKx1rTzfmRLkeCgJzcbBnOV2dkhAPc8cCeT6agocpjg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/opener": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", @@ -15430,6 +16290,12 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/os-browserify": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==", + "dev": true + }, "node_modules/os-dns-native": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/os-dns-native/-/os-dns-native-1.2.1.tgz", @@ -15470,6 +16336,15 @@ "node": ">=8" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -15497,6 +16372,47 @@ "node": ">=8" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -16028,21 +16944,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/pretty-format": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.6.2.tgz", - "integrity": "sha512-7AeGuCYNGmycyQbCqd/3PWH4eOoX/OiCa0uphp57NVTeAGdJGaAliecxwBDHYQCIvrW7aDBZCYeNTP/WX69mkg==", - "dev": true, - "dependencies": { - "@jest/types": "^26.6.2", - "ansi-regex": "^5.0.0", - "ansi-styles": "^4.0.0", - "react-is": "^17.0.1" - }, - "engines": { - "node": ">= 10" - } - }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -16112,6 +17013,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -16162,6 +17069,16 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -16176,6 +17093,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystring-es3": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", + "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==", + "dev": true, + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -16737,6 +17663,15 @@ "node": ">=8" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -16747,6 +17682,22 @@ "node": ">=0.10.0" } }, + "node_modules/rewiremock": { + "version": "3.14.5", + "resolved": "https://registry.npmjs.org/rewiremock/-/rewiremock-3.14.5.tgz", + "integrity": "sha512-MdPutvaUd+kKVz/lcEz6N6337s4PxRUR5vhphIp2/TJRgfXIckomIkCsIAbwB53MjiSLwi7KBMdQ9lPWE5WpYA==", + "dev": true, + "dependencies": { + "babel-runtime": "^6.26.0", + "compare-module-exports": "^2.1.0", + "lodash.some": "^4.6.0", + "lodash.template": "^4.4.0", + "node-libs-browser": "^2.1.0", + "path-parse": "^1.0.5", + "wipe-node-cache": "^2.1.2", + "wipe-webpack-cache": "^2.1.0" + } + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -16818,107 +17769,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/run-applescript/node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" + "node": ">=0.12.0" } }, - "node_modules/run-applescript/node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-applescript/node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/run-applescript/node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-applescript/node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/run-applescript/node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "engines": { - "node": ">=6" - } - }, - "node_modules/run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" } }, "node_modules/rxjs": { @@ -16972,6 +17852,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -17025,6 +17914,19 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/seek-bzip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", @@ -17050,9 +17952,9 @@ "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "bin": { "semver": "bin/semver.js" }, @@ -17142,14 +18044,16 @@ } }, "node_modules/set-function-length": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz", - "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", "dependencies": { - "define-data-property": "^1.1.1", - "get-intrinsic": "^1.2.1", + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.0" + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -17233,13 +18137,17 @@ } }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -17295,6 +18203,21 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + }, "node_modules/sinon": { "version": "9.2.4", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.2.4.tgz", @@ -17632,6 +18555,15 @@ "nan": "^2.18.0" } }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -17761,6 +18693,55 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/stream-http": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz", + "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "dev": true, + "dependencies": { + "builtin-status-codes": "^3.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.3.6", + "to-arraybuffer": "^1.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/stream-http/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/stream-http/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/stream-http/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/stream-http/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -17901,6 +18882,15 @@ "node": ">=8" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-dirs": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", @@ -18337,6 +19327,12 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "dev": true + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -18373,6 +19369,18 @@ "node": ">=0.4" } }, + "node_modules/timers-browserify": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", + "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "dev": true, + "dependencies": { + "setimmediate": "^1.0.4" + }, + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/tiny-emitter": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", @@ -18398,6 +19406,12 @@ "node": ">=14.14" } }, + "node_modules/to-arraybuffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", + "dev": true + }, "node_modules/to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", @@ -18431,6 +19445,12 @@ "node": ">=0.6" } }, + "node_modules/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==", + "dev": true + }, "node_modules/totalist": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", @@ -18477,6 +19497,15 @@ "node": ">=18" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "dev": true, + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ts-loader": { "version": "9.5.1", "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.5.1.tgz", @@ -18598,6 +19627,12 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, + "node_modules/tty-browserify": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==", + "dev": true + }, "node_modules/tunnel": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", @@ -18670,6 +19705,32 @@ "node": ">= 0.6" } }, + "node_modules/typechat": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/typechat/-/typechat-0.0.10.tgz", + "integrity": "sha512-iF/wLLaZWt4Q9WO8stpq3NKilAa4b8hnCD16EirdhaxzAYk80MCb1wnW1il7GhkMNJuhJUD38dxs8q4A/EdxJw==", + "dev": true, + "dependencies": { + "axios": "^1.4.0", + "typescript": "^5.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typechat/node_modules/typescript": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/typed-array-length": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz", @@ -18715,10 +19776,11 @@ } }, "node_modules/uc.micro": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", - "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", - "dev": true + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "dev": true, + "license": "MIT" }, "node_modules/unbox-primitive": { "version": "1.0.2", @@ -18850,6 +19912,19 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.4.tgz", + "integrity": "sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==", + "dev": true, + "dependencies": { + "punycode": "^1.4.1", + "qs": "^6.12.3" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/url-join": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", @@ -18866,6 +19941,27 @@ "requires-port": "^1.0.0" } }, + "node_modules/url/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", + "dev": true + }, + "node_modules/url/node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/use-sync-external-store": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.2.tgz", @@ -18875,11 +19971,26 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dev": true, + "dependencies": { + "inherits": "2.0.3" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -18920,6 +20031,12 @@ "node": ">= 0.8" } }, + "node_modules/vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", + "dev": true + }, "node_modules/vscode-jsonrpc": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.1.0.tgz", @@ -19324,27 +20441,114 @@ "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true }, - "node_modules/win-export-certificate-and-key": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/win-export-certificate-and-key/-/win-export-certificate-and-key-2.1.0.tgz", - "integrity": "sha512-WeMLa/2uNZcS/HWGKU2G1Gzeh3vHpV/UFvwLhJLKxPHYFAbubxxVcJbqmPXaqySWK1Ymymh16zKK5WYIJ3zgzA==", - "hasInstallScript": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], + "node_modules/winston": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.14.2.tgz", + "integrity": "sha512-CO8cdpBB2yqzEf8v895L+GNKYJiEq8eKlHU38af3snQBQ+sdAIUepjMSguOIJC7ICbzm0ZI+Af2If4vIJrtmOg==", + "dev": true, "dependencies": { - "bindings": "^1.5.0", - "node-addon-api": "^3.1.0" + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.6.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" } }, - "node_modules/win-export-certificate-and-key/node_modules/node-addon-api": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", - "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", - "license": "MIT", - "optional": true + "node_modules/winston-transport": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.1.tgz", + "integrity": "sha512-wQCXXVgfv/wUPOfb2x0ruxzwkcZfxcktz6JIMUaPLmcNhO4bZTwA/WtDWK74xV3F2dKu8YadrFv0qhwYjVEwhA==", + "dev": true, + "dependencies": { + "logform": "^2.6.1", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston-transport/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/winston/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/winston/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/wipe-node-cache": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/wipe-node-cache/-/wipe-node-cache-2.1.2.tgz", + "integrity": "sha512-m7NXa8qSxBGMtdQilOu53ctMaIBXy93FOP04EC1Uf4bpsE+r+adfLKwIMIvGbABsznaSNxK/ErD4xXDyY5og9w==", + "dev": true + }, + "node_modules/wipe-webpack-cache": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wipe-webpack-cache/-/wipe-webpack-cache-2.1.0.tgz", + "integrity": "sha512-OXzQMGpA7MnQQ8AG+uMl5mWR2ezy6fw1+DMHY+wzYP1qkF1jrek87psLBmhZEj+er4efO/GD4R8jXWFierobaA==", + "dev": true, + "dependencies": { + "wipe-node-cache": "^2.1.0" + } }, "node_modules/word-wrap": { "version": "1.2.5", @@ -19629,18 +20833,6 @@ "node": ">=10" } }, - "node_modules/yargs-unparser/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/yargs-unparser/node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -19697,6 +20889,15 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zod-to-json-schema": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz", + "integrity": "sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==", + "dev": true, + "peerDependencies": { + "zod": "^3.23.3" + } } } } diff --git a/package.json b/package.json index 17c3a8c23..3e5970436 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,9 @@ }, "publisher": "mongodb", "categories": [ + "AI", + "Chat", + "Data Science", "Programming Languages", "Snippets", "Other" @@ -32,6 +35,7 @@ "color": "#3D4F58", "theme": "dark" }, + "enabledApiProposals": [], "license": "SEE LICENSE IN LICENSE.txt", "main": "./dist/extension.js", "scripts": { @@ -40,7 +44,7 @@ "update-grammar": "ts-node ./scripts/update-grammar.ts", "precompile": "npm run clean", "compile": "npm-run-all compile:*", - "compile:keyfile": "ts-node ./scripts/generate-keyfile.ts", + "compile:constants": "ts-node ./scripts/generate-constants.ts", "compile:resources": "npm run update-grammar", "compile:extension": "tsc -p ./", "compile:extension-bundles": "webpack --mode development", @@ -51,8 +55,9 @@ "test": "npm run test-webview && npm run test-extension", "test-extension": "cross-env NODE_OPTIONS=--no-force-async-hooks-checks xvfb-maybe node ./out/test/runTest.js", "test-webview": "mocha -r ts-node/register --file ./src/test/setup-webview.ts src/test/suite/views/webview-app/**/*.test.tsx", + "ai-accuracy-tests": "env TS_NODE_FILES=true mocha -r ts-node/register --file ./src/test/ai-accuracy-tests/test-setup.ts ./src/test/ai-accuracy-tests/ai-accuracy-tests.ts", "analyze-bundle": "webpack --mode production --analyze", - "vscode:prepublish": "npm run clean && npm run compile:keyfile && npm run compile:resources && webpack --mode production", + "vscode:prepublish": "npm run clean && npm run compile:constants && npm run compile:resources && webpack --mode production", "check": "npm run lint && npm run depcheck", "depcheck": "depcheck", "package": "cross-env NODE_OPTIONS='--require ./scripts/no-npm-list-fail.js' vsce package --githubBranch main", @@ -73,11 +78,37 @@ }, "activationEvents": [ "onView:mongoDB", + "onStartupFinished", "onLanguage:json", "onLanguage:javascript", "onLanguage:plaintext" ], "contributes": { + "chatParticipants": [ + { + "id": "mongodb.participant", + "name": "MongoDB", + "description": "Ask anything about MongoDB, from writing queries to questions about your cluster.", + "isSticky": true, + "commands": [ + { + "name": "query", + "isSticky": true, + "description": "Ask how to write MongoDB queries or pipelines. For example, you can ask: \"Show me all the documents where the address contains the word street\"." + }, + { + "name": "docs", + "isSticky": true, + "description": "Ask MongoDB-related questions and find answers in the official documentation." + }, + { + "name": "schema", + "isSticky": true, + "description": "Analyze a collection's schema." + } + ] + } + ], "viewsContainers": { "activitybar": [ { @@ -144,6 +175,30 @@ } ], "commands": [ + { + "command": "mdb.selectDatabaseWithParticipant", + "title": "MongoDB: Select Database with Participant" + }, + { + "command": "mdb.selectCollectionWithParticipant", + "title": "MongoDB: Select Collection with Participant" + }, + { + "command": "mdb.participantViewRawSchemaOutput", + "title": "MongoDB: View Raw Schema JSON Output" + }, + { + "command": "mdb.connectWithParticipant", + "title": "MongoDB: Change Active Connection with Participant" + }, + { + "command": "mdb.runParticipantCode", + "title": "Run Content Generated by Participant" + }, + { + "command": "mdb.openParticipantCodeInPlayground", + "title": "Open Generated by Participant Content In Playground" + }, { "command": "mdb.connect", "title": "MongoDB: Connect" @@ -164,6 +219,10 @@ "command": "mdb.openOverviewPage", "title": "MongoDB: Open Overview Page" }, + { + "command": "mdb.openMongoDBIssueReporter", + "title": "MongoDB: Open MongoDB Issue Reporter" + }, { "command": "mdb.openMongoDBShell", "title": "MongoDB: Launch MongoDB Shell" @@ -690,6 +749,34 @@ } ], "commandPalette": [ + { + "command": "mdb.selectDatabaseWithParticipant", + "when": "false" + }, + { + "command": "mdb.selectCollectionWithParticipant", + "when": "false" + }, + { + "command": "mdb.participantViewRawSchemaOutput", + "when": "false" + }, + { + "command": "mdb.connectWithParticipant", + "when": "false" + }, + { + "command": "mdb.runParticipantCode", + "when": "false" + }, + { + "command": "mdb.openParticipantCodeInPlayground", + "when": "false" + }, + { + "command": "mdb.runParticipantCode", + "when": "false" + }, { "command": "mdb.disconnect", "when": "mdb.connectedToMongoDB == true" @@ -758,6 +845,10 @@ "command": "mdb.createNewPlaygroundFromOverviewPage", "when": "false" }, + { + "command": "mdb.openMongoDBIssueReporter", + "when": "true" + }, { "command": "mdb.createNewPlaygroundFromTreeView", "when": "false" @@ -1024,6 +1115,16 @@ "default": true, "description": "Show a confirmation message before running commands in a playground." }, + "mdb.confirmRunCopilotCode": { + "type": "boolean", + "default": true, + "description": "Show a confirmation message before running code generated by the MongoDB participant." + }, + "mdb.useSampleDocsInCopilot": { + "type": "boolean", + "default": false, + "description": "Enable sending sample field values with the VSCode copilot chat @MongoDB participant /query command." + }, "mdb.confirmDeleteDocument": { "type": "boolean", "default": true, @@ -1120,6 +1221,7 @@ "@mongodb-js/oidc-mock-provider": "^0.9.1", "@mongodb-js/oidc-plugin": "^0.4.0", "@mongodb-js/prettier-config-devtools": "^1.0.1", + "mongodb-rag-core": "^0.4.1", "@mongodb-js/sbom-tools": "^0.7.1", "@mongodb-js/signing-utils": "^0.3.5", "@mongosh/service-provider-core": "^2.2.15", @@ -1129,7 +1231,6 @@ "@types/chai": "^4.3.17", "@types/debug": "^4.1.12", "@types/glob": "^7.2.0", - "@types/jest": "^26.0.24", "@types/micromatch": "^4.0.9", "@types/mkdirp": "^2.0.0", "@types/mocha": "^8.2.3", @@ -1143,7 +1244,7 @@ "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", "@vscode/test-electron": "^2.4.1", - "@vscode/vsce": "^2.31.1", + "@vscode/vsce": "^3.1.0", "buffer": "^6.0.3", "chai": "^4.5.0", "chai-as-promised": "^7.1.2", @@ -1167,11 +1268,13 @@ "node-fetch": "^2.7.0", "node-loader": "^0.6.0", "npm-run-all": "^4.1.5", + "openai": "^4.55.7", "ora": "^5.4.1", "path-browserify": "^1.0.1", "pre-commit": "^1.2.2", "prettier": "^2.8.8", "process": "^0.11.10", + "rewiremock": "^3.14.5", "sinon": "^9.2.4", "sinon-chai": "^3.7.0", "source-map-support": "^0.5.21", diff --git a/resources/screenshots/mongodb-participant-commands.png b/resources/screenshots/mongodb-participant-commands.png new file mode 100644 index 000000000..ab5e9319a Binary files /dev/null and b/resources/screenshots/mongodb-participant-commands.png differ diff --git a/resources/screenshots/mongodb-participant.png b/resources/screenshots/mongodb-participant.png new file mode 100644 index 000000000..69f3d6e2f Binary files /dev/null and b/resources/screenshots/mongodb-participant.png differ diff --git a/scripts/generate-constants.ts b/scripts/generate-constants.ts new file mode 100644 index 000000000..e84d60000 --- /dev/null +++ b/scripts/generate-constants.ts @@ -0,0 +1,24 @@ +#! /usr/bin/env ts-node + +import ora from 'ora'; +import fs from 'fs'; +import path from 'path'; +import { resolve } from 'path'; +import { config } from 'dotenv'; +import { promisify } from 'util'; + +const writeFile = promisify(fs.writeFile); +const ROOT_DIR = path.join(__dirname, '..'); +const ui = ora('Generate constants file').start(); + +config({ path: resolve(__dirname, '../.env') }); + +(async () => { + await writeFile( + `${ROOT_DIR}/constants.json`, + JSON.stringify({ segmentKey: process.env.SEGMENT_KEY }, null, 2) + ); + ui.succeed('The constants file has been generated'); +})().catch((error) => { + ui.fail(`Failed to generate constants file: ${error.message}`); +}); diff --git a/scripts/generate-keyfile.ts b/scripts/generate-keyfile.ts deleted file mode 100644 index b7a6add73..000000000 --- a/scripts/generate-keyfile.ts +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/env ts-node - -import ora from 'ora'; -import fs from 'fs'; -import path from 'path'; -import { resolve } from 'path'; -import { config } from 'dotenv'; -import { promisify } from 'util'; - -const writeFile = promisify(fs.writeFile); -const ROOT_DIR = path.join(__dirname, '..'); -const ui = ora('Generate constants keyfile').start(); - -config({ path: resolve(__dirname, '../.env') }); - -(async () => { - if (process.env.SEGMENT_KEY) { - await writeFile( - `${ROOT_DIR}/constants.json`, - JSON.stringify({ segmentKey: process.env.SEGMENT_KEY }, null, 2) - ); - ui.succeed('Generated segment constants file'); - } else { - throw new Error('The Segment key is missing in environment variables'); - } -})().catch((error) => { - ui.fail(`Failed to generate segment constants file: ${error.message}`); -}); diff --git a/src/commands/index.ts b/src/commands/index.ts index ebef4dec1..c36c477d2 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -5,6 +5,8 @@ enum EXTENSION_COMMANDS { MDB_DISCONNECT = 'mdb.disconnect', MDB_REMOVE_CONNECTION = 'mdb.removeConnection', + OPEN_MONGODB_ISSUE_REPORTER = 'mdb.openMongoDBIssueReporter', + MDB_OPEN_MDB_SHELL = 'mdb.openMongoDBShell', MDB_OPEN_MDB_SHELL_FROM_TREE_VIEW = 'mdb.treeViewOpenMongoDBShell', @@ -73,6 +75,14 @@ enum EXTENSION_COMMANDS { MDB_START_STREAM_PROCESSOR = 'mdb.startStreamProcessor', MDB_STOP_STREAM_PROCESSOR = 'mdb.stopStreamProcessor', MDB_DROP_STREAM_PROCESSOR = 'mdb.dropStreamProcessor', + + // Chat participant. + OPEN_PARTICIPANT_CODE_IN_PLAYGROUND = 'mdb.openParticipantCodeInPlayground', + RUN_PARTICIPANT_CODE = 'mdb.runParticipantCode', + CONNECT_WITH_PARTICIPANT = 'mdb.connectWithParticipant', + SELECT_DATABASE_WITH_PARTICIPANT = 'mdb.selectDatabaseWithParticipant', + SELECT_COLLECTION_WITH_PARTICIPANT = 'mdb.selectCollectionWithParticipant', + PARTICIPANT_OPEN_RAW_SCHEMA_OUTPUT = 'mdb.participantViewRawSchemaOutput', } export default EXTENSION_COMMANDS; diff --git a/src/connectionController.ts b/src/connectionController.ts index ddedadc4d..0ca1a6b20 100644 --- a/src/connectionController.ts +++ b/src/connectionController.ts @@ -79,7 +79,12 @@ function isOIDCAuth(connectionString: string): boolean { // Exported for testing. export function getNotifyDeviceFlowForConnectionAttempt( connectionOptions: ConnectionOptions -) { +): + | ((deviceFlowInformation: { + verificationUrl: string; + userCode: string; + }) => void) + | undefined { const isOIDCConnectionAttempt = isOIDCAuth( connectionOptions.connectionString ); @@ -97,7 +102,7 @@ export function getNotifyDeviceFlowForConnectionAttempt( }: { verificationUrl: string; userCode: string; - }) => { + }): void => { void vscode.window.showInformationMessage( `Visit the following URL to complete authentication: ${verificationUrl} Enter the following code on that page: ${userCode}` ); @@ -381,7 +386,7 @@ export default class ConnectionController { ...cloneDeep(connectionOptions.oidc), openBrowser: browserAuthCommand ? { command: browserAuthCommand } - : async ({ signal, url }) => { + : async ({ signal, url }): Promise => { try { await openLink(url); } catch (err) { @@ -419,7 +424,14 @@ export default class ConnectionController { } log.info('Successfully connected', { connectionId }); - void vscode.window.showInformationMessage('MongoDB connection successful.'); + + const message = 'MongoDB connection successful.'; + this._statusView.showMessage(message); + setTimeout(() => { + if (this._statusView._statusBarItem.text === message) { + this._statusView.hideMessage(); + } + }, 5000); dataService.addReauthenticationHandler( this._reauthenticationHandler.bind(this) @@ -428,7 +440,12 @@ export default class ConnectionController { this._currentConnectionId = connectionId; this._connectionAttempt = null; this._connectingConnectionId = null; + + this._connections[connectionId].lastUsed = new Date(); this.eventEmitter.emit(DataServiceEventTypes.ACTIVE_CONNECTION_CHANGED); + await this._connectionStorage.saveConnection( + this._connections[connectionId] + ); // Send metrics to Segment this.sendTelemetry(dataService, connectionType); @@ -457,7 +474,7 @@ export default class ConnectionController { } // Used to re-authenticate with OIDC. - async _reauthenticationHandler() { + async _reauthenticationHandler(): Promise { const removeConfirmationResponse = await vscode.window.showInformationMessage( 'You need to re-authenticate to the database in order to continue.', @@ -476,7 +493,7 @@ export default class ConnectionController { }: { connectionInfo: LoadedConnection; dataService: DataService; - }) { + }): Promise { if (connectionInfo.storageLocation === 'NONE') { return; } @@ -501,7 +518,7 @@ export default class ConnectionController { // ?. because mocks in tests don't provide it dataService.on?.('connectionInfoSecretsChanged', () => { - void (async () => { + void (async (): Promise => { try { if ( !vscode.workspace.getConfiguration('mdb').get('persistOIDCTokens') @@ -534,7 +551,7 @@ export default class ConnectionController { }); } - cancelConnectionAttempt() { + cancelConnectionAttempt(): void { this._connectionAttempt?.cancelConnectionAttempt(); } @@ -603,10 +620,16 @@ export default class ConnectionController { 'mdb.isAtlasStreams', false ); - void vscode.window.showInformationMessage('MongoDB disconnected.'); this._disconnecting = false; - this._statusView.hideMessage(); + + const message = 'MongoDB disconnected.'; + this._statusView.showMessage(message); + setTimeout(() => { + if (this._statusView._statusBarItem.text === message) { + this._statusView.hideMessage(); + } + }, 5000); return true; } @@ -788,11 +811,11 @@ export default class ConnectionController { this.eventEmitter.removeListener(eventType, listener); } - deactivate() { + deactivate(): void { this.eventEmitter.removeAllListeners(); } - closeConnectionStringInput() { + closeConnectionStringInput(): void { this._connectionStringInputCancellationToken?.cancel(); } @@ -887,7 +910,7 @@ export default class ConnectionController { return connectionStringData.toString(); } - isConnectedToAtlasStreams() { + isConnectedToAtlasStreams(): boolean { return ( this.isCurrentlyConnected() && isAtlasStream(this.getActiveConnectionString()) @@ -909,7 +932,7 @@ export default class ConnectionController { return connectionString; } - getActiveDataService() { + getActiveDataService(): DataService | null { return this._activeDataService; } @@ -1036,7 +1059,7 @@ export default class ConnectionController { ); if (!selectedQuickPickItem) { - return true; + return false; } if (selectedQuickPickItem.data.type === NewConnectionType.NEW_CONNECTION) { @@ -1044,7 +1067,7 @@ export default class ConnectionController { } if (!selectedQuickPickItem.data.connectionId) { - return true; + return false; } const { successfullyConnected } = await this.connectWithConnectionId( diff --git a/src/editors/editDocumentCodeLensProvider.ts b/src/editors/editDocumentCodeLensProvider.ts index 723df8deb..c380c95ec 100644 --- a/src/editors/editDocumentCodeLensProvider.ts +++ b/src/editors/editDocumentCodeLensProvider.ts @@ -33,7 +33,7 @@ export default class EditDocumentCodeLensProvider content: Document; namespace: string | null; uri: vscode.Uri; - }) { + }): void { let resultCodeLensesInfo: EditDocumentInfo[] = []; resultCodeLensesInfo = this._updateCodeLensesForCursor({ @@ -44,7 +44,7 @@ export default class EditDocumentCodeLensProvider this._codeLensesInfo[data.uri.toString()] = resultCodeLensesInfo; } - updateCodeLensesForPlayground(playgroundResult: PlaygroundResult) { + updateCodeLensesForPlayground(playgroundResult: PlaygroundResult): void { const source = DocumentSource.DOCUMENT_SOURCE_PLAYGROUND; let resultCodeLensesInfo: EditDocumentInfo[] = []; diff --git a/src/editors/mongoDBDocumentService.ts b/src/editors/mongoDBDocumentService.ts index 928d02de0..1806cddda 100644 --- a/src/editors/mongoDBDocumentService.ts +++ b/src/editors/mongoDBDocumentService.ts @@ -98,13 +98,11 @@ export default class MongoDBDocumentService { returnDocument: 'after', } ); - - this._statusView.hideMessage(); this._telemetryService.trackDocumentUpdated(source, true); } catch (error) { - this._statusView.hideMessage(); - return this._saveDocumentFailed(formatError(error).message); + } finally { + this._statusView.hideMessage(); } } @@ -141,17 +139,15 @@ export default class MongoDBDocumentService { { limit: 1 } ); - this._statusView.hideMessage(); - if (!documents || documents.length === 0) { return; } return getEJSON(documents[0]); } catch (error) { - this._statusView.hideMessage(); - return this._fetchDocumentFailed(formatError(error).message); + } finally { + this._statusView.hideMessage(); } } } diff --git a/src/editors/playgroundController.ts b/src/editors/playgroundController.ts index 4378e80b8..118a7a376 100644 --- a/src/editors/playgroundController.ts +++ b/src/editors/playgroundController.ts @@ -15,6 +15,7 @@ import { DatabaseTreeItem } from '../explorer'; import type ExportToLanguageCodeLensProvider from './exportToLanguageCodeLensProvider'; import formatError from '../utils/formatError'; import type { LanguageServerController } from '../language'; +import playgroundBasicTextTemplate from '../templates/playgroundBasicTextTemplate'; import playgroundCreateIndexTemplate from '../templates/playgroundCreateIndexTemplate'; import playgroundCreateCollectionTemplate from '../templates/playgroundCreateCollectionTemplate'; import playgroundCloneDocumentTemplate from '../templates/playgroundCloneDocumentTemplate'; @@ -58,10 +59,14 @@ interface ToCompile { let dummySandbox; +function getActiveEditorFilePath(): string | undefined { + return vscode.window.activeTextEditor?.document.uri.fsPath; +} + // TODO: this function was copied from the compass-export-to-language module // https://github.com/mongodb-js/compass/blob/7c4bc0789a7b66c01bb7ba63955b3b11ed40c094/packages/compass-export-to-language/src/modules/count-aggregation-stages-in-string.js // and should be updated as well when the better solution for the problem will be found. -const countAggregationStagesInString = (str: string) => { +const countAggregationStagesInString = (str: string): number => { if (!dummySandbox) { dummySandbox = vm.createContext(Object.create(null), { codeGeneration: { strings: false, wasm: false }, @@ -111,6 +116,9 @@ const exportModeMapping: Record< [ExportToLanguageMode.OTHER]: undefined, }; +const connectBeforeRunningMessage = + 'Please connect to a database before running a playground.'; + /** * This controller manages playground. */ @@ -132,8 +140,6 @@ export default class PlaygroundController { private _playgroundResultViewProvider: PlaygroundResultProvider; private _activeConnectionChangedHandler: () => void; - private _codeToEvaluate = ''; - constructor({ connectionController, languageServerController, @@ -161,7 +167,7 @@ export default class PlaygroundController { this._playgroundSelectedCodeActionProvider = playgroundSelectedCodeActionProvider; - this._activeConnectionChangedHandler = () => { + this._activeConnectionChangedHandler = (): void => { void this._activeConnectionChanged(); }; this._connectionController.addEventListener( @@ -171,7 +177,7 @@ export default class PlaygroundController { const onDidChangeActiveTextEditor = ( editor: vscode.TextEditor | undefined - ) => { + ): void => { if (editor?.document.uri.scheme === PLAYGROUND_RESULT_SCHEME) { this._playgroundResultViewColumn = editor.viewColumn; this._playgroundResultTextDocument = editor?.document; @@ -374,6 +380,21 @@ export default class PlaygroundController { return this._createPlaygroundFileWithContent(content); } + createPlaygroundFromParticipantCode({ + text, + }: { + text: string; + }): Promise { + const useDefaultTemplate = !!vscode.workspace + .getConfiguration('mdb') + .get('useDefaultTemplateForPlayground'); + const content = useDefaultTemplate + ? playgroundBasicTextTemplate.replace('PLAYGROUND_CONTENT', text) + : text; + this._telemetryService.trackPlaygroundCreated('agent'); + return this._createPlaygroundFileWithContent(content); + } + createPlaygroundForCloneDocument( documentContents: string, databaseName: string, @@ -424,13 +445,17 @@ export default class PlaygroundController { return this._createPlaygroundFileWithContent(content); } - async _evaluate(codeToEvaluate: string): Promise { + async _evaluate({ + codeToEvaluate, + filePath, + }: { + codeToEvaluate: string; + filePath?: string; + }): Promise { const connectionId = this._connectionController.getActiveConnectionId(); if (!connectionId) { - throw new Error( - 'Please connect to a database before running a playground.' - ); + throw new Error(connectBeforeRunningMessage); } this._statusView.showMessage('Getting results...'); @@ -441,7 +466,7 @@ export default class PlaygroundController { result = await this._languageServerController.evaluate({ codeToEvaluate, connectionId, - filePath: vscode.window.activeTextEditor?.document.uri.fsPath, + filePath, }); } catch (error) { const msg = @@ -468,11 +493,15 @@ export default class PlaygroundController { return this._activeTextEditor?.document.getText(selection) || ''; } - async _evaluateWithCancelModal(): Promise { + async _evaluateWithCancelModal({ + codeToEvaluate, + filePath, + }: { + codeToEvaluate: string; + filePath?: string; + }): Promise { if (!this._connectionController.isCurrentlyConnected()) { - throw new Error( - 'Please connect to a database before running a playground.' - ); + throw new Error(connectBeforeRunningMessage); } try { @@ -491,9 +520,10 @@ export default class PlaygroundController { }); // Run all playground scripts. - const result: ShellEvaluateResult = await this._evaluate( - this._codeToEvaluate - ); + const result: ShellEvaluateResult = await this._evaluate({ + codeToEvaluate, + filePath, + }); return result; } @@ -507,10 +537,8 @@ export default class PlaygroundController { } } - async _openPlaygroundResult(): Promise { - this._playgroundResultViewProvider.setPlaygroundResult( - this._playgroundResult - ); + async _openInResultPane(result: PlaygroundResult): Promise { + this._playgroundResultViewProvider.setPlaygroundResult(result); if (!this._playgroundResultTextDocument) { await this._openResultAsVirtualDocument(); @@ -521,7 +549,7 @@ export default class PlaygroundController { await this._showResultAsVirtualDocument(); if (this._playgroundResultTextDocument) { - const language = this._playgroundResult?.language || 'plaintext'; + const language = result?.language || 'plaintext'; await vscode.languages.setTextDocumentLanguage( this._playgroundResultTextDocument, @@ -558,15 +586,58 @@ export default class PlaygroundController { } } - async _evaluatePlayground(): Promise { + async evaluateParticipantCode(codeToEvaluate: string): Promise { + const shouldConfirmRunCopilotCode = vscode.workspace + .getConfiguration('mdb') + .get('confirmRunCopilotCode'); + + if (!this._connectionController.isCurrentlyConnected()) { + // TODO(VSCODE-618): Prompt user to connect when clicked. + void vscode.window.showErrorMessage(connectBeforeRunningMessage); + + return false; + } + + if (shouldConfirmRunCopilotCode === true) { + const name = this._connectionController.getActiveConnectionName(); + const confirmRunCopilotCode = await vscode.window.showInformationMessage( + `Are you sure you want to run this code generated by the MongoDB participant against ${name}? This confirmation can be disabled in the extension settings.`, + { modal: true }, + 'Yes' + ); + + if (confirmRunCopilotCode !== 'Yes') { + return false; + } + } + + const evaluateResponse: ShellEvaluateResult = + await this._evaluateWithCancelModal({ + codeToEvaluate, + }); + + if (!evaluateResponse || !evaluateResponse.result) { + return false; + } + + await this._openInResultPane(evaluateResponse.result); + + return true; + } + + async _evaluatePlayground({ + codeToEvaluate, + filePath, + }: { + codeToEvaluate: string; + filePath?: string; + }): Promise { const shouldConfirmRunAll = vscode.workspace .getConfiguration('mdb') .get('confirmRunAll'); if (!this._connectionController.isCurrentlyConnected()) { - void vscode.window.showErrorMessage( - 'Please connect to a database before running a playground.' - ); + void vscode.window.showErrorMessage(connectBeforeRunningMessage); return false; } @@ -585,15 +656,17 @@ export default class PlaygroundController { } const evaluateResponse: ShellEvaluateResult = - await this._evaluateWithCancelModal(); + await this._evaluateWithCancelModal({ + codeToEvaluate, + filePath, + }); if (!evaluateResponse || !evaluateResponse.result) { return false; } this._playgroundResult = evaluateResponse.result; - - await this._openPlaygroundResult(); + await this._openInResultPane(this._playgroundResult); return true; } @@ -608,9 +681,11 @@ export default class PlaygroundController { } this._isPartialRun = true; - this._codeToEvaluate = this._selectedText; - return this._evaluatePlayground(); + return this._evaluatePlayground({ + codeToEvaluate: this._selectedText || '', + filePath: getActiveEditorFilePath(), + }); } runAllPlaygroundBlocks(): Promise { @@ -626,9 +701,11 @@ export default class PlaygroundController { } this._isPartialRun = false; - this._codeToEvaluate = this._getAllText(); - return this._evaluatePlayground(); + return this._evaluatePlayground({ + codeToEvaluate: this._getAllText(), + filePath: getActiveEditorFilePath(), + }); } runAllOrSelectedPlaygroundBlocks(): Promise { @@ -643,28 +720,26 @@ export default class PlaygroundController { return Promise.resolve(false); } - const selections = this._activeTextEditor.selections; - - if ( - !selections || - !Array.isArray(selections) || - (selections.length === 1 && this._getSelectedText(selections[0]) === '') - ) { + let codeToEvaluate = ''; + if (!this._selectedText) { this._isPartialRun = false; - this._codeToEvaluate = this._getAllText(); - } else if (this._selectedText) { + codeToEvaluate = this._getAllText(); + } else { this._isPartialRun = true; - this._codeToEvaluate = this._selectedText; + codeToEvaluate = this._selectedText; } - return this._evaluatePlayground(); + return this._evaluatePlayground({ + codeToEvaluate, + filePath: getActiveEditorFilePath(), + }); } async fixThisInvalidInteractiveSyntax({ documentUri, range, fix, - }: ThisDiagnosticFix) { + }: ThisDiagnosticFix): Promise { const edit = new vscode.WorkspaceEdit(); edit.replace(documentUri, range, fix); await vscode.workspace.applyEdit(edit); @@ -674,7 +749,7 @@ export default class PlaygroundController { async fixAllInvalidInteractiveSyntax({ documentUri, diagnostics, - }: AllDiagnosticFixes) { + }: AllDiagnosticFixes): Promise { const edit = new vscode.WorkspaceEdit(); for (const { range, fix } of diagnostics) { @@ -848,7 +923,7 @@ export default class PlaygroundController { language, num_stages: selectedText ? countAggregationStagesInString(selectedText) - : null, + : undefined, with_import_statements: importStatements, with_builders: builders, with_driver_syntax: driverSyntax, @@ -870,7 +945,7 @@ export default class PlaygroundController { } /* eslint-enable camelcase */ - await this._openPlaygroundResult(); + await this._openInResultPane(this._playgroundResult); } catch (error) { log.error(`Export to the '${language}' language failed`, error); const printableError = formatError(error); diff --git a/src/explorer/connectionTreeItem.ts b/src/explorer/connectionTreeItem.ts index 7eecddc72..9c47423a4 100644 --- a/src/explorer/connectionTreeItem.ts +++ b/src/explorer/connectionTreeItem.ts @@ -118,7 +118,6 @@ export default class ConnectionTreeItem const dbs = await dataService.listDatabases({ nameOnly: true, }); - return dbs.map((dbItem) => dbItem.name); } catch (error) { throw new Error( diff --git a/src/language/worker.ts b/src/language/worker.ts index 316e9f3e5..3a1d1605d 100644 --- a/src/language/worker.ts +++ b/src/language/worker.ts @@ -26,7 +26,7 @@ const getContent = ({ type, printable }: EvaluationResult) => { : getEJSON(printable); }; -const getLanguage = (evaluationResult: EvaluationResult) => { +export const getLanguage = (evaluationResult: EvaluationResult) => { const content = getContent(evaluationResult); if (typeof content === 'object' && content !== null) { @@ -40,14 +40,27 @@ type ExecuteCodeOptions = { codeToEvaluate: string; connectionString: string; connectionOptions: MongoClientOptions; + onPrint?: (values: EvaluationResult[]) => void; filePath?: string; }; +function handleEvalPrint(values: EvaluationResult[]) { + parentPort?.postMessage({ + name: ServerCommands.SHOW_CONSOLE_OUTPUT, + payload: values.map((v) => { + return typeof v.printable === 'string' + ? v.printable + : util.inspect(v.printable); + }), + }); +} + /** * Execute code from a playground. */ -const execute = async ({ +export const execute = async ({ codeToEvaluate, + onPrint = handleEvalPrint, connectionString, connectionOptions, filePath, @@ -66,16 +79,7 @@ const execute = async ({ // Collect console.log() output. runtime.setEvaluationListener({ - onPrint(values: EvaluationResult[]) { - parentPort?.postMessage({ - name: ServerCommands.SHOW_CONSOLE_OUTPUT, - payload: values.map((v) => { - return typeof v.printable === 'string' - ? v.printable - : util.inspect(v.printable); - }), - }); - }, + onPrint, }); // In order to support local require directly from the file where code lives, we can not wrap the diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index f49ee48d4..e481a57ea 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -42,6 +42,12 @@ import WebviewController from './views/webviewController'; import { createIdFactory, generateId } from './utils/objectIdHelper'; import { ConnectionStorage } from './storage/connectionStorage'; import type StreamProcessorTreeItem from './explorer/streamProcessorTreeItem'; +import type { + ParticipantCommand, + RunParticipantCodeCommandArgs, +} from './participant/participant'; +import ParticipantController from './participant/participant'; +import type { OpenSchemaCommandArgs } from './participant/prompts/schema'; // This class is the top-level controller for our extension. // Commands which the extensions handles are defined in the function `activate`. @@ -65,6 +71,7 @@ export default class MDBExtensionController implements vscode.Disposable { _activeConnectionCodeLensProvider: ActiveConnectionCodeLensProvider; _editDocumentCodeLensProvider: EditDocumentCodeLensProvider; _exportToLanguageCodeLensProvider: ExportToLanguageCodeLensProvider; + _participantController: ParticipantController; constructor( context: vscode.ExtensionContext, @@ -117,6 +124,11 @@ export default class MDBExtensionController implements vscode.Disposable { playgroundSelectedCodeActionProvider: this._playgroundSelectedCodeActionProvider, }); + this._participantController = new ParticipantController({ + connectionController: this._connectionController, + storageController: this._storageController, + telemetryService: this._telemetryService, + }); this._editorsController = new EditorsController({ context, connectionController: this._connectionController, @@ -145,6 +157,7 @@ export default class MDBExtensionController implements vscode.Disposable { this._helpExplorer.activateHelpTreeView(this._telemetryService); this._playgroundsExplorer.activatePlaygroundsTreeView(); this._telemetryService.activateSegmentAnalytics(); + this._participantController.createParticipant(this._context); await this._connectionController.loadSavedConnections(); await this._languageServerController.startLanguageServer(); @@ -179,6 +192,19 @@ export default class MDBExtensionController implements vscode.Disposable { this._connectionController.changeActiveConnection() ); + this.registerCommand( + EXTENSION_COMMANDS.OPEN_MONGODB_ISSUE_REPORTER, + async () => { + return await vscode.commands.executeCommand( + 'workbench.action.openIssueReporter', + { + extensionId: 'mongodb.mongodb-vscode', + issueSource: 'extension', + } + ); + } + ); + // ------ SHELL ------ // this.registerCommand(EXTENSION_COMMANDS.MDB_OPEN_MDB_SHELL, () => launchMongoShell(this._connectionController) @@ -267,6 +293,78 @@ export default class MDBExtensionController implements vscode.Disposable { this.registerEditorCommands(); this.registerTreeViewCommands(); + + // ------ CHAT PARTICIPANT ------ // + this.registerParticipantCommand( + EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, + ({ runnableContent }: RunParticipantCodeCommandArgs) => { + return this._playgroundController.createPlaygroundFromParticipantCode({ + text: runnableContent, + }); + } + ); + this.registerParticipantCommand( + EXTENSION_COMMANDS.RUN_PARTICIPANT_CODE, + ({ runnableContent }: RunParticipantCodeCommandArgs) => { + return this._playgroundController.evaluateParticipantCode( + runnableContent + ); + } + ); + this.registerCommand( + EXTENSION_COMMANDS.CONNECT_WITH_PARTICIPANT, + (data: { id?: string; command?: string }) => { + return this._participantController.connectWithParticipant(data); + } + ); + this.registerCommand( + EXTENSION_COMMANDS.SELECT_DATABASE_WITH_PARTICIPANT, + (data: { + chatId: string; + command: ParticipantCommand; + databaseName?: string; + }) => { + return this._participantController.selectDatabaseWithParticipant(data); + } + ); + this.registerCommand( + EXTENSION_COMMANDS.SELECT_COLLECTION_WITH_PARTICIPANT, + (data: any) => { + return this._participantController.selectCollectionWithParticipant( + data + ); + } + ); + this.registerCommand( + EXTENSION_COMMANDS.PARTICIPANT_OPEN_RAW_SCHEMA_OUTPUT, + async ({ schema }: OpenSchemaCommandArgs) => { + const document = await vscode.workspace.openTextDocument({ + language: 'json', + content: schema, + }); + await vscode.window.showTextDocument(document, { preview: true }); + + return !!document; + } + ); + }; + + registerParticipantCommand = ( + command: string, + commandHandler: (...args: any[]) => Promise + ): void => { + const commandHandlerWithTelemetry = (args: any[]): Promise => { + this._telemetryService.trackCommandRun(command); + + return commandHandler(args); + }; + const participant = this._participantController.getParticipant(); + if (participant) { + this._context.subscriptions.push( + participant, + vscode.commands.registerCommand(command, commandHandlerWithTelemetry) + ); + } }; registerCommand = ( @@ -708,7 +806,7 @@ export default class MDBExtensionController implements vscode.Disposable { this.registerAtlasStreamsTreeViewCommands(); } - registerAtlasStreamsTreeViewCommands() { + registerAtlasStreamsTreeViewCommands(): void { this.registerCommand( EXTENSION_COMMANDS.MDB_ADD_STREAM_PROCESSOR, async (element: ConnectionTreeItem): Promise => { diff --git a/src/participant/chatMetadata.ts b/src/participant/chatMetadata.ts new file mode 100644 index 000000000..1462c4ae4 --- /dev/null +++ b/src/participant/chatMetadata.ts @@ -0,0 +1,42 @@ +import * as vscode from 'vscode'; +import { v4 as uuidv4 } from 'uuid'; + +export type ChatMetadata = { + databaseName?: string; + collectionName?: string; + docsChatbotConversationId?: string; +}; + +export class ChatMetadataStore { + _chats: { [chatId: string]: ChatMetadata } = {}; + + constructor() {} + + setChatMetadata(chatId: string, metadata: ChatMetadata): void { + this._chats[chatId] = metadata; + } + + getChatMetadata(chatId: string): ChatMetadata | undefined { + return this._chats[chatId]; + } + + // Exposed for stubbing in tests. + static createNewChatId(): string { + return uuidv4(); + } + + static getChatIdFromHistoryOrNewChatId( + history: ReadonlyArray + ): string { + for (const historyItem of history) { + if ( + historyItem instanceof vscode.ChatResponseTurn && + historyItem.result?.metadata?.chatId + ) { + return historyItem.result.metadata.chatId; + } + } + + return ChatMetadataStore.createNewChatId(); + } +} diff --git a/src/participant/constants.ts b/src/participant/constants.ts new file mode 100644 index 000000000..90fd81490 --- /dev/null +++ b/src/participant/constants.ts @@ -0,0 +1,125 @@ +import type * as vscode from 'vscode'; +import { ChatMetadataStore } from './chatMetadata'; + +export const CHAT_PARTICIPANT_ID = 'mongodb.participant'; +export const CHAT_PARTICIPANT_MODEL = 'gpt-4o'; + +export type ParticipantResponseType = + | 'query' + | 'schema' + | 'docs' + | 'generic' + | 'emptyRequest' + | 'cancelledRequest' + | 'askToConnect' + | 'askForNamespace'; + +export const codeBlockIdentifier = { + start: '```javascript', + end: '```', +}; + +interface Metadata { + intent: Exclude; + chatId: string; +} + +interface AskForNamespaceMetadata { + intent: 'askForNamespace'; + chatId: string; + databaseName?: string | undefined; + collectionName?: string | undefined; +} + +interface DocsRequestMetadata { + intent: 'docs'; + chatId: string; + docsChatbotMessageId?: string; +} + +export interface ChatResult extends vscode.ChatResult { + readonly metadata: Metadata | AskForNamespaceMetadata | DocsRequestMetadata; +} + +export function namespaceRequestChatResult({ + databaseName, + collectionName, + history, +}: { + history: ReadonlyArray; + databaseName: string | undefined; + collectionName: string | undefined; +}): ChatResult { + return { + metadata: { + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId(history), + intent: 'askForNamespace', + databaseName, + collectionName, + }, + }; +} + +export function createCancelledRequestChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('cancelledRequest', history); +} + +function createChatResult( + intent: ParticipantResponseType, + history: ReadonlyArray +): ChatResult { + return { + metadata: { + intent, + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId(history), + }, + }; +} + +export function emptyRequestChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('emptyRequest', history); +} + +export function askToConnectChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('askToConnect', history); +} + +export function genericRequestChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('generic', history); +} + +export function queryRequestChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('query', history); +} + +export function docsRequestChatResult({ + chatId, + docsChatbotMessageId, +}: { + chatId: string; + docsChatbotMessageId?: string; +}): ChatResult { + return { + metadata: { + chatId, + intent: 'docs', + docsChatbotMessageId, + }, + }; +} + +export function schemaRequestChatResult( + history: ReadonlyArray +): ChatResult { + return createChatResult('schema', history); +} diff --git a/src/participant/docsChatbotAIService.ts b/src/participant/docsChatbotAIService.ts new file mode 100644 index 000000000..47858280f --- /dev/null +++ b/src/participant/docsChatbotAIService.ts @@ -0,0 +1,207 @@ +import type { Reference, VerifiedAnswer } from 'mongodb-rag-core'; + +const MONGODB_DOCS_CHATBOT_BASE_URI = 'https://knowledge.mongodb.com/'; + +const MONGODB_DOCS_CHATBOT_API_VERSION = 'v1'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { version } = require('../../package.json'); + +type Role = 'user' | 'assistant'; + +type ConversationData = { + _id: string; + createdAt: string; + messages: MessageData[]; + conversationId: string; +}; + +type MessageData = { + id: string; + role: Role; + content: string; + createdAt: string; + rating?: boolean; + references?: Reference[]; + suggestedPrompts?: string[]; + metadata?: AssistantMessageMetadata; +}; + +type AssistantMessageMetadata = { + [k: string]: unknown; + + /** + If the message came from the verified answers collection, contains the + metadata about the verified answer. + */ + verifiedAnswer?: { + _id: VerifiedAnswer['_id']; + created: string; + updated: string | undefined; + }; +}; + +export class DocsChatbotAIService { + _serverBaseUri: string; + + constructor() { + this._serverBaseUri = + process.env.MONGODB_DOCS_CHATBOT_BASE_URI_OVERRIDE || + MONGODB_DOCS_CHATBOT_BASE_URI; + } + + private getUri(path: string): string { + return `${this._serverBaseUri}api/${MONGODB_DOCS_CHATBOT_API_VERSION}${path}`; + } + + _fetch({ + uri, + method, + body, + signal, + headers, + }: { + uri: string; + method: string; + signal?: AbortSignal; + body?: string; + headers?: { [key: string]: string }; + }): Promise { + return fetch(uri, { + headers: { + origin: this._serverBaseUri, + 'User-Agent': `mongodb-vscode/${version}`, + ...headers, + }, + method, + signal, + ...(body && { body }), + }); + } + + async createConversation({ + signal, + }: { + signal: AbortSignal; + }): Promise { + const uri = this.getUri('/conversations'); + const res = await this._fetch({ + uri, + method: 'POST', + signal, + }); + + let data; + try { + data = await res.json(); + } catch (error) { + throw new Error('[Docs chatbot] Internal server error'); + } + + if (res.status === 400) { + throw new Error(`[Docs chatbot] Bad request: ${data.error}`); + } + if (res.status === 429) { + throw new Error(`[Docs chatbot] Rate limited: ${data.error}`); + } + if (res.status >= 500) { + throw new Error( + `[Docs chatbot] Internal server error: ${ + data.error ? data.error : `${res.status} - ${res.statusText}}` + }` + ); + } + + return { + ...data, + conversationId: data._id, + }; + } + + async addMessage({ + conversationId, + message, + signal, + }: { + conversationId: string; + message: string; + signal: AbortSignal; + }): Promise { + const uri = this.getUri(`/conversations/${conversationId}/messages`); + const res = await this._fetch({ + uri, + method: 'POST', + body: JSON.stringify({ message }), + headers: { 'Content-Type': 'application/json' }, + signal, + }); + + let data; + try { + data = await res.json(); + } catch (error) { + throw new Error('[Docs chatbot] Internal server error'); + } + + if (res.status === 400) { + throw new Error(`[Docs chatbot] Bad request: ${data.error}`); + } + if (res.status === 404) { + throw new Error(`[Docs chatbot] Conversation not found: ${data.error}`); + } + if (res.status === 429) { + throw new Error(`[Docs chatbot] Rate limited: ${data.error}`); + } + if (res.status === 504) { + throw new Error(`[Docs chatbot] Timeout: ${data.error}`); + } + if (res.status >= 500) { + throw new Error( + `[Docs chatbot] Internal server error: ${ + data.error ? data.error : `${res.status} - ${res.statusText}}` + }` + ); + } + + return data; + } + + async rateMessage({ + conversationId, + messageId, + rating, + }: { + conversationId: string; + messageId: string; + rating: boolean; + }): Promise { + const uri = this.getUri( + `/conversations/${conversationId}/messages/${messageId}/rating` + ); + const res = await this._fetch({ + uri, + method: 'POST', + body: JSON.stringify({ rating }), + headers: { 'Content-Type': 'application/json' }, + }); + + if (res.status === 204) { + return rating; + } + + let data; + if (res.status >= 400) { + try { + data = await res.json(); + } catch (error) { + throw new Error(`[Docs chatbot] Internal server error: ${error}`); + } + } + + throw new Error( + `[Docs chatbot] Internal server error: ${ + data.error ? data.error : `${res.status} - ${res.statusText}}` + }` + ); + } +} diff --git a/src/participant/markdown.ts b/src/participant/markdown.ts new file mode 100644 index 000000000..73af90b85 --- /dev/null +++ b/src/participant/markdown.ts @@ -0,0 +1,23 @@ +import * as vscode from 'vscode'; + +export function createMarkdownLink({ + commandId, + data, + name, +}: { + commandId: string; + // TODO: Create types for this data so we can also then use them on the extension + // controller when we parse the result. + data: { + [field: string]: any; + }; + name: string; +}): vscode.MarkdownString { + const encodedData = encodeURIComponent(JSON.stringify(data)); + const commandQueryString = data ? `?${encodedData}` : ''; + const link = new vscode.MarkdownString( + `- [${name}](command:${commandId}${commandQueryString})\n` + ); + link.isTrusted = { enabledCommands: [commandId] }; + return link; +} diff --git a/src/participant/model.ts b/src/participant/model.ts new file mode 100644 index 000000000..f5c2568d0 --- /dev/null +++ b/src/participant/model.ts @@ -0,0 +1,23 @@ +import * as vscode from 'vscode'; + +import { CHAT_PARTICIPANT_MODEL } from './constants'; + +let model: vscode.LanguageModelChat; + +export async function getCopilotModel(): Promise< + vscode.LanguageModelChat | undefined +> { + if (!model) { + try { + const [model] = await vscode.lm.selectChatModels({ + vendor: 'copilot', + family: CHAT_PARTICIPANT_MODEL, + }); + return model; + } catch (err) { + // Model is not ready yet. It is being initialised with the first user prompt. + } + } + + return; +} diff --git a/src/participant/participant.ts b/src/participant/participant.ts new file mode 100644 index 000000000..79c3f16bd --- /dev/null +++ b/src/participant/participant.ts @@ -0,0 +1,1574 @@ +import * as vscode from 'vscode'; +import { getSimplifiedSchema, parseSchema } from 'mongodb-schema'; +import type { Document } from 'bson'; +import type { Reference } from 'mongodb-rag-core'; +import util from 'util'; + +import { createLogger } from '../logging'; +import type ConnectionController from '../connectionController'; +import type { LoadedConnection } from '../storage/connectionStorage'; +import EXTENSION_COMMANDS from '../commands'; +import type { StorageController } from '../storage'; +import { StorageVariables } from '../storage'; +import { Prompts } from './prompts'; +import type { ChatResult } from './constants'; +import { + askToConnectChatResult, + CHAT_PARTICIPANT_ID, + emptyRequestChatResult, + genericRequestChatResult, + namespaceRequestChatResult, + queryRequestChatResult, + docsRequestChatResult, + schemaRequestChatResult, + createCancelledRequestChatResult, + codeBlockIdentifier, +} from './constants'; +import { SchemaFormatter } from './schema'; +import { getSimplifiedSampleDocuments } from './sampleDocuments'; +import { getCopilotModel } from './model'; +import { createMarkdownLink } from './markdown'; +import { ChatMetadataStore } from './chatMetadata'; +import { + DOCUMENTS_TO_SAMPLE_FOR_SCHEMA_PROMPT, + type OpenSchemaCommandArgs, +} from './prompts/schema'; +import { + chatResultFeedbackKindToTelemetryValue, + ParticipantErrorTypes, + TelemetryEventTypes, +} from '../telemetry/telemetryService'; +import { DocsChatbotAIService } from './docsChatbotAIService'; +import type TelemetryService from '../telemetry/telemetryService'; +import type { ModelInput } from './prompts/promptBase'; +import { processStreamWithIdentifiers } from './streamParsing'; +import type { PromptIntent } from './prompts/intent'; + +const log = createLogger('participant'); + +const NUM_DOCUMENTS_TO_SAMPLE = 3; + +const MONGODB_DOCS_LINK = 'https://www.mongodb.com/docs/'; + +interface NamespaceQuickPicks { + label: string; + data: string; +} + +export type RunParticipantCodeCommandArgs = { + runnableContent: string; +}; + +export type ParticipantCommand = '/query' | '/schema' | '/docs'; + +const MAX_MARKDOWN_LIST_LENGTH = 10; + +export default class ParticipantController { + _participant?: vscode.ChatParticipant; + _connectionController: ConnectionController; + _storageController: StorageController; + _chatMetadataStore: ChatMetadataStore; + _docsChatbotAIService: DocsChatbotAIService; + _telemetryService: TelemetryService; + + constructor({ + connectionController, + storageController, + telemetryService, + }: { + connectionController: ConnectionController; + storageController: StorageController; + telemetryService: TelemetryService; + }) { + this._connectionController = connectionController; + this._storageController = storageController; + this._chatMetadataStore = new ChatMetadataStore(); + this._telemetryService = telemetryService; + this._docsChatbotAIService = new DocsChatbotAIService(); + } + + createParticipant(context: vscode.ExtensionContext): vscode.ChatParticipant { + // Chat participants appear as top-level options in the chat input + // when you type `@`, and can contribute sub-commands in the chat input + // that appear when you type `/`. + this._participant = vscode.chat.createChatParticipant( + CHAT_PARTICIPANT_ID, + this.chatHandler.bind(this) + ); + this._participant.iconPath = vscode.Uri.joinPath( + vscode.Uri.parse(context.extensionPath), + 'images', + 'mongodb.png' + ); + log.info('Chat participant created', { + participantId: this._participant?.id, + }); + this._participant.onDidReceiveFeedback(this.handleUserFeedback.bind(this)); + return this._participant; + } + + getParticipant(): vscode.ChatParticipant | undefined { + return this._participant; + } + + handleError(err: any, command: string): never { + let errorCode: string | undefined; + let errorName: ParticipantErrorTypes; + // Making the chat request might fail because + // - model does not exist + // - user consent not given + // - quote limits exceeded + if (err instanceof vscode.LanguageModelError) { + errorCode = err.code; + } + + if (err instanceof Error) { + // Unwrap the error if a cause is provided + err = err.cause || err; + } + + const message: string = err.message || err.toString(); + + if (message.includes('off_topic')) { + errorName = ParticipantErrorTypes.CHAT_MODEL_OFF_TOPIC; + } else if (message.includes('Filtered by Responsible AI Service')) { + errorName = ParticipantErrorTypes.FILTERED; + } else if (message.includes('Prompt failed validation')) { + errorName = ParticipantErrorTypes.INVALID_PROMPT; + } else { + errorName = ParticipantErrorTypes.OTHER; + } + + log.error('Participant encountered an error', { + command, + error_code: errorCode, + error_name: errorName, + }); + + this._telemetryService.track( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED, + { + command, + error_code: errorCode, + error_name: errorName, + } + ); + + // Re-throw other errors so they show up in the UI. + throw err; + } + + /** + * In order to get access to the model, and to write more messages to the chat after + * an async event that occurs after we've already completed our response, we need + * to be handling a chat request. This could be when a user clicks a button or link + * in the chat. To work around this, we can write a message as the user, which will + * trigger the chat handler and give us access to the model. + */ + writeChatMessageAsUser(message: string): Thenable { + return vscode.commands.executeCommand('workbench.action.chat.open', { + query: `@MongoDB ${message}`, + }); + } + + async _getChatResponse({ + modelInput, + token, + }: { + modelInput: ModelInput; + token: vscode.CancellationToken; + }): Promise { + const model = await getCopilotModel(); + + if (!model) { + throw new Error('Copilot model not found'); + } + + log.info('Sending request to model', { + messages: modelInput.messages.map( + (message: vscode.LanguageModelChatMessage) => + util.inspect({ + role: message.role, + contentLength: message.content.length, + }) + ), + }); + this._telemetryService.trackCopilotParticipantPrompt(modelInput.stats); + + const modelResponse = await model.sendRequest( + modelInput.messages, + {}, + token + ); + + log.info('Model response received'); + + return modelResponse; + } + + async streamChatResponse({ + modelInput, + stream, + token, + }: { + modelInput: ModelInput; + stream: vscode.ChatResponseStream; + token: vscode.CancellationToken; + }): Promise<{ outputLength: number }> { + const chatResponse = await this._getChatResponse({ + modelInput, + token, + }); + + let length = 0; + for await (const fragment of chatResponse.text) { + stream.markdown(fragment); + length += fragment.length; + } + + return { + outputLength: length, + }; + } + + _streamCodeBlockActions({ + runnableContent, + stream, + }: { + runnableContent: string; + stream: vscode.ChatResponseStream; + }): void { + runnableContent = runnableContent.trim(); + + if (!runnableContent) { + return; + } + + const commandArgs: RunParticipantCodeCommandArgs = { + runnableContent, + }; + stream.button({ + command: EXTENSION_COMMANDS.RUN_PARTICIPANT_CODE, + title: vscode.l10n.t('▶️ Run'), + arguments: [commandArgs], + }); + stream.button({ + command: EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, + title: vscode.l10n.t('Open in playground'), + arguments: [commandArgs], + }); + } + + async streamChatResponseContentWithCodeActions({ + modelInput, + stream, + token, + }: { + modelInput: ModelInput; + stream: vscode.ChatResponseStream; + token: vscode.CancellationToken; + }): Promise<{ + outputLength: number; + hasCodeBlock: boolean; + }> { + const chatResponse = await this._getChatResponse({ + modelInput, + token, + }); + + let outputLength = 0; + let hasCodeBlock = false; + await processStreamWithIdentifiers({ + processStreamFragment: (fragment: string) => { + stream.markdown(fragment); + outputLength += fragment.length; + }, + onStreamIdentifier: (content: string) => { + this._streamCodeBlockActions({ runnableContent: content, stream }); + hasCodeBlock = true; + }, + inputIterable: chatResponse.text, + identifier: codeBlockIdentifier, + }); + + log.info('Streamed response to chat', { + outputLength, + hasCodeBlock, + }); + + return { + outputLength, + hasCodeBlock, + }; + } + + // This will stream all of the response content and create a string from it. + // It should only be used when the entire response is needed at one time. + async getChatResponseContent({ + modelInput, + token, + }: { + modelInput: ModelInput; + token: vscode.CancellationToken; + }): Promise { + let responseContent = ''; + const chatResponse = await this._getChatResponse({ + modelInput, + token, + }); + for await (const fragment of chatResponse.text) { + responseContent += fragment; + } + + return responseContent; + } + + async _handleRoutedGenericRequest( + request: vscode.ChatRequest, + context: vscode.ChatContext, + stream: vscode.ChatResponseStream, + token: vscode.CancellationToken + ): Promise { + const modelInput = await Prompts.generic.buildMessages({ + request, + context, + connectionNames: this._getConnectionNames(), + }); + + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + token, + stream, + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'generic', + has_cta: false, + found_namespace: false, + has_runnable_content: hasCodeBlock, + output_length: outputLength, + }); + + return genericRequestChatResult(context.history); + } + + async _routeRequestToHandler({ + context, + promptIntent, + request, + stream, + token, + }: { + context: vscode.ChatContext; + promptIntent: Omit; + request: vscode.ChatRequest; + stream: vscode.ChatResponseStream; + token: vscode.CancellationToken; + }): Promise { + switch (promptIntent) { + case 'Query': + return this.handleQueryRequest(request, context, stream, token); + case 'Docs': + return this.handleDocsRequest(request, context, stream, token); + case 'Schema': + return this.handleSchemaRequest(request, context, stream, token); + case 'Code': + return this.handleQueryRequest(request, context, stream, token); + default: + return this._handleRoutedGenericRequest( + request, + context, + stream, + token + ); + } + } + + async _getIntentFromChatRequest({ + context, + request, + token, + }: { + context: vscode.ChatContext; + request: vscode.ChatRequest; + token: vscode.CancellationToken; + }): Promise { + const modelInput = await Prompts.intent.buildMessages({ + connectionNames: this._getConnectionNames(), + request, + context, + }); + + const responseContent = await this.getChatResponseContent({ + modelInput, + token, + }); + + log.info('Received intent response from model', { + responseContentLength: responseContent.length, + }); + + return Prompts.intent.getIntentFromModelResponse(responseContent); + } + + async handleGenericRequest( + request: vscode.ChatRequest, + context: vscode.ChatContext, + stream: vscode.ChatResponseStream, + token: vscode.CancellationToken + ): Promise { + // We "prompt chain" to handle the generic requests. + // First we ask the model to parse for intent. + // If there is an intent, we can route it to one of the handlers (/commands). + // When there is no intention or it's generic we handle it with a generic handler. + const promptIntent = await this._getIntentFromChatRequest({ + context, + request, + token, + }); + + if (token.isCancellationRequested) { + return this._handleCancelledRequest({ + context, + stream, + }); + } + + return this._routeRequestToHandler({ + context, + promptIntent, + request, + stream, + token, + }); + } + + async connectWithParticipant({ + id, + command, + }: { + id?: string; + command?: string; + }): Promise { + if (!id) { + const didChangeActiveConnection = + await this._connectionController.changeActiveConnection(); + if (!didChangeActiveConnection) { + // If they don't choose a connection then we can't proceed; + return false; + } + } else { + await this._connectionController.connectWithConnectionId(id); + } + + const connectionName = this._connectionController.getActiveConnectionName(); + + return this.writeChatMessageAsUser( + `${command ? `${command} ` : ''}${connectionName}` + ) as Promise; + } + + getConnectionsTree(command: ParticipantCommand): vscode.MarkdownString[] { + return [ + ...this._connectionController + .getSavedConnections() + .sort((a, b) => { + const aTime = a.lastUsed ? new Date(a.lastUsed).getTime() : 0; + const bTime = b.lastUsed ? new Date(b.lastUsed).getTime() : 0; + return bTime - aTime; + }) + .slice(0, MAX_MARKDOWN_LIST_LENGTH) + .map((conn: LoadedConnection) => + createMarkdownLink({ + commandId: EXTENSION_COMMANDS.CONNECT_WITH_PARTICIPANT, + data: { + id: conn.id, + command, + }, + name: conn.name, + }) + ), + createMarkdownLink({ + commandId: EXTENSION_COMMANDS.CONNECT_WITH_PARTICIPANT, + name: 'Show more', + data: { + command, + }, + }), + ]; + } + + async getDatabaseQuickPicks( + command: ParticipantCommand + ): Promise { + const dataService = this._connectionController.getActiveDataService(); + if (!dataService) { + // Run a blank command to get the user to connect first. + void this.writeChatMessageAsUser(command); + return []; + } + + try { + const databases = await dataService.listDatabases({ + nameOnly: true, + }); + return databases.map((db) => ({ + label: db.name, + data: db.name, + })); + } catch (error) { + return []; + } + } + + async _selectDatabaseWithQuickPick( + command: ParticipantCommand + ): Promise { + const databases = await this.getDatabaseQuickPicks(command); + const selectedQuickPickItem = await vscode.window.showQuickPick(databases, { + placeHolder: 'Select a database...', + }); + return selectedQuickPickItem?.data; + } + + async selectDatabaseWithParticipant({ + chatId, + command, + databaseName: _databaseName, + }: { + chatId: string; + command: ParticipantCommand; + databaseName?: string; + }): Promise { + let databaseName: string | undefined = _databaseName; + if (!databaseName) { + databaseName = await this._selectDatabaseWithQuickPick(command); + if (!databaseName) { + return false; + } + } + + this._chatMetadataStore.setChatMetadata(chatId, { + databaseName: databaseName, + }); + + return this.writeChatMessageAsUser( + `${command} ${databaseName}` + ) as Promise; + } + + async getCollectionQuickPicks({ + command, + databaseName, + }: { + command: ParticipantCommand; + databaseName: string; + }): Promise { + const dataService = this._connectionController.getActiveDataService(); + if (!dataService) { + // Run a blank command to get the user to connect first. + void this.writeChatMessageAsUser(command); + return []; + } + + try { + const collections = await dataService.listCollections(databaseName); + return collections.map((db) => ({ + label: db.name, + data: db.name, + })); + } catch (error) { + return []; + } + } + + async _selectCollectionWithQuickPick({ + command, + databaseName, + }: { + command: ParticipantCommand; + databaseName: string; + }): Promise { + const collections = await this.getCollectionQuickPicks({ + command, + databaseName, + }); + const selectedQuickPickItem = await vscode.window.showQuickPick( + collections, + { + placeHolder: 'Select a collection...', + } + ); + return selectedQuickPickItem?.data; + } + + async selectCollectionWithParticipant({ + command, + chatId, + databaseName, + collectionName: _collectionName, + }: { + command: ParticipantCommand; + chatId: string; + databaseName: string; + collectionName?: string; + }): Promise { + let collectionName: string | undefined = _collectionName; + if (!collectionName) { + collectionName = await this._selectCollectionWithQuickPick({ + command, + databaseName, + }); + if (!collectionName) { + return false; + } + } + + this._chatMetadataStore.setChatMetadata(chatId, { + databaseName: databaseName, + collectionName: collectionName, + }); + return this.writeChatMessageAsUser( + `${command} ${collectionName}` + ) as Promise; + } + + async renderDatabasesTree({ + command, + context, + stream, + }: { + command: ParticipantCommand; + context: vscode.ChatContext; + stream: vscode.ChatResponseStream; + }): Promise { + const dataService = this._connectionController.getActiveDataService(); + if (!dataService) { + return; + } + + stream.push( + new vscode.ChatResponseProgressPart('Fetching database names...') + ); + + try { + const databases = await dataService.listDatabases({ + nameOnly: true, + }); + databases.slice(0, MAX_MARKDOWN_LIST_LENGTH).forEach((db) => + stream.markdown( + createMarkdownLink({ + commandId: EXTENSION_COMMANDS.SELECT_DATABASE_WITH_PARTICIPANT, + data: { + command, + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ), + databaseName: db.name, + }, + name: db.name, + }) + ) + ); + if (databases.length > MAX_MARKDOWN_LIST_LENGTH) { + stream.markdown( + createMarkdownLink({ + data: { + command, + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ), + }, + commandId: EXTENSION_COMMANDS.SELECT_DATABASE_WITH_PARTICIPANT, + name: 'Show more', + }) + ); + } + } catch (error) { + log.error('Unable to fetch databases:', error); + + // Users can always do this manually when asked to provide a database name. + return; + } + } + + async renderCollectionsTree({ + command, + context, + databaseName, + stream, + }: { + command: ParticipantCommand; + databaseName: string; + context: vscode.ChatContext; + stream: vscode.ChatResponseStream; + }): Promise { + const dataService = this._connectionController.getActiveDataService(); + if (!dataService) { + return; + } + + stream.push( + new vscode.ChatResponseProgressPart('Fetching collection names...') + ); + + try { + const collections = await dataService.listCollections(databaseName); + collections.slice(0, MAX_MARKDOWN_LIST_LENGTH).forEach((coll) => + stream.markdown( + createMarkdownLink({ + commandId: EXTENSION_COMMANDS.SELECT_COLLECTION_WITH_PARTICIPANT, + data: { + command, + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ), + databaseName, + collectionName: coll.name, + }, + name: coll.name, + }) + ) + ); + if (collections.length > MAX_MARKDOWN_LIST_LENGTH) { + stream.markdown( + createMarkdownLink({ + commandId: EXTENSION_COMMANDS.SELECT_COLLECTION_WITH_PARTICIPANT, + data: { + command, + chatId: ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ), + databaseName, + }, + name: 'Show more', + }) + ); + } + } catch (error) { + log.error('Unable to fetch collections:', error); + + // Users can always do this manually when asked to provide a collection name. + return; + } + } + + async _getNamespaceFromChat({ + request, + context, + token, + }: { + request: vscode.ChatRequest; + context: vscode.ChatContext; + token: vscode.CancellationToken; + }): Promise<{ + databaseName: string | undefined; + collectionName: string | undefined; + }> { + const messagesWithNamespace = await Prompts.namespace.buildMessages({ + context, + request, + connectionNames: this._getConnectionNames(), + }); + + let { + databaseName, + collectionName, + }: { + databaseName: string | undefined; + collectionName: string | undefined; + } = { + databaseName: undefined, + collectionName: undefined, + }; + + // When there's no user message content we can + // skip the request to the model. This would happen with /schema. + if (Prompts.doMessagesContainUserInput(messagesWithNamespace.messages)) { + // VSCODE-626: When there's an empty message sent to the ai model, + // it currently errors (not on insiders, only main VSCode). + // Here we're defaulting to have some content as a workaround. + // TODO: Remove this when the issue is fixed. + messagesWithNamespace.messages[ + messagesWithNamespace.messages.length - 1 + ].content = + messagesWithNamespace.messages[ + messagesWithNamespace.messages.length - 1 + ].content.trim() || 'see previous messages'; + + const responseContentWithNamespace = await this.getChatResponseContent({ + modelInput: messagesWithNamespace, + token, + }); + ({ databaseName, collectionName } = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse( + responseContentWithNamespace + )); + } + + // See if there's a namespace set in the + // chat metadata we can fallback to if the model didn't find it. + const chatId = ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ); + const { + databaseName: databaseNameFromMetadata, + collectionName: collectionNameFromMetadata, + } = this._chatMetadataStore.getChatMetadata(chatId) ?? {}; + + log.info('Namespaces found in chat', { + databaseName: databaseName || databaseNameFromMetadata, + collectionName: collectionName || collectionNameFromMetadata, + }); + + return { + databaseName: databaseName || databaseNameFromMetadata, + collectionName: collectionName || collectionNameFromMetadata, + }; + } + + async _askForNamespace({ + command, + context, + databaseName, + collectionName, + stream, + }: { + command: ParticipantCommand; + context: vscode.ChatContext; + databaseName: string | undefined; + collectionName: string | undefined; + stream: vscode.ChatResponseStream; + }): Promise { + // If no database or collection name is found in the user prompt, + // we retrieve the available namespaces from the current connection. + // Users can then select a value by clicking on an item in the list. + if (!databaseName) { + stream.markdown( + `What is the name of the database you would like${ + command === '/query' ? ' this query' : '' + } to run against?\n\n` + ); + await this.renderDatabasesTree({ + command, + context, + stream, + }); + } else if (!collectionName) { + stream.markdown( + `Which collection would you like to use within ${databaseName}?\n\n` + ); + await this.renderCollectionsTree({ + command, + databaseName, + context, + stream, + }); + } + + return namespaceRequestChatResult({ + databaseName, + collectionName, + history: context.history, + }); + } + + _doesLastMessageAskForNamespace( + history: ReadonlyArray + ): boolean { + const lastMessageMetaData = history[ + history.length - 1 + ] as vscode.ChatResponseTurn; + + return ( + (lastMessageMetaData?.result as ChatResult)?.metadata?.intent === + 'askForNamespace' + ); + } + + _askToConnect({ + command, + context, + stream, + }: { + command: ParticipantCommand; + context: vscode.ChatContext; + stream: vscode.ChatResponseStream; + }): ChatResult { + log.info('Participant asked user to connect'); + + stream.markdown( + "Looks like you aren't currently connected, first let's get you connected to the cluster we'd like to create this query to run against.\n\n" + ); + + const tree = this.getConnectionsTree(command); + for (const item of tree) { + stream.markdown(item); + } + return askToConnectChatResult(context.history); + } + + _handleCancelledRequest({ + context, + stream, + }: { + context: vscode.ChatContext; + stream: vscode.ChatResponseStream; + }): ChatResult { + stream.markdown('\nRequest cancelled.'); + + return createCancelledRequestChatResult(context.history); + } + + // The sample documents returned from this are simplified (strings and arrays shortened). + // The sample documents are only returned when a user has the setting enabled. + async _fetchCollectionSchemaAndSampleDocuments({ + databaseName, + collectionName, + amountOfDocumentsToSample = NUM_DOCUMENTS_TO_SAMPLE, + schemaFormat = 'simplified', + token, + stream, + }: { + databaseName: string; + collectionName: string; + amountOfDocumentsToSample?: number; + schemaFormat?: 'simplified' | 'full'; + token: vscode.CancellationToken; + stream: vscode.ChatResponseStream; + }): Promise<{ + schema?: string; + sampleDocuments?: Document[]; + amountOfDocumentsSampled: number; + }> { + const dataService = this._connectionController.getActiveDataService(); + if (!dataService) { + return { + amountOfDocumentsSampled: 0, + }; + } + + stream.push( + new vscode.ChatResponseProgressPart( + 'Fetching documents and analyzing schema...' + ) + ); + + const abortController = new AbortController(); + token.onCancellationRequested(() => { + abortController.abort(); + }); + + try { + const sampleDocuments = await dataService.sample( + `${databaseName}.${collectionName}`, + { + query: {}, + size: amountOfDocumentsToSample, + }, + { promoteValues: false, maxTimeMS: 10_000 }, + { + abortSignal: abortController.signal, + } + ); + + if (!sampleDocuments) { + return { + amountOfDocumentsSampled: 0, + }; + } + + let schema: string; + if (schemaFormat === 'simplified') { + const unformattedSchema = await getSimplifiedSchema(sampleDocuments); + schema = new SchemaFormatter().format(unformattedSchema); + } else { + const unformattedSchema = await parseSchema(sampleDocuments, { + storeValues: false, + }); + schema = JSON.stringify(unformattedSchema, null, 2); + } + + const useSampleDocsInCopilot = !!vscode.workspace + .getConfiguration('mdb') + .get('useSampleDocsInCopilot'); + + return { + sampleDocuments: useSampleDocsInCopilot + ? getSimplifiedSampleDocuments(sampleDocuments) + : undefined, + schema, + amountOfDocumentsSampled: sampleDocuments.length, + }; + } catch (err: any) { + log.error('Unable to fetch schema and sample documents:', err); + throw err; + } + } + + async handleEmptyNamespaceMessage({ + command, + context, + stream, + }: { + command: ParticipantCommand; + context: vscode.ChatContext; + stream: vscode.ChatResponseStream; + }): Promise { + const lastMessageMetaData: vscode.ChatResponseTurn | undefined = context + .history[context.history.length - 1] as vscode.ChatResponseTurn; + const lastMessage = lastMessageMetaData?.result as ChatResult; + if (lastMessage?.metadata?.intent !== 'askForNamespace') { + stream.markdown(Prompts.generic.getEmptyRequestResponse()); + return emptyRequestChatResult(context.history); + } + + // When the last message was asking for a database or collection name, + // we re-ask the question. + const databaseName = lastMessage.metadata.databaseName; + if (databaseName) { + stream.markdown( + vscode.l10n.t( + 'Please select a collection by either clicking on an item in the list or typing the name manually in the chat.' + ) + ); + await this.renderCollectionsTree({ + command, + databaseName, + context, + stream, + }); + } else { + stream.markdown( + vscode.l10n.t( + 'Please select a database by either clicking on an item in the list or typing the name manually in the chat.' + ) + ); + await this.renderDatabasesTree({ + command, + context, + stream, + }); + } + + return namespaceRequestChatResult({ + databaseName, + collectionName: undefined, + history: context.history, + }); + } + + // @MongoDB /schema + async handleSchemaRequest( + request: vscode.ChatRequest, + context: vscode.ChatContext, + stream: vscode.ChatResponseStream, + token: vscode.CancellationToken + ): Promise { + if (!this._connectionController.getActiveDataService()) { + return this._askToConnect({ + command: '/schema', + context, + stream, + }); + } + + if ( + Prompts.isPromptEmpty(request) && + this._doesLastMessageAskForNamespace(context.history) + ) { + return this.handleEmptyNamespaceMessage({ + command: '/schema', + context, + stream, + }); + } + + const { databaseName, collectionName } = await this._getNamespaceFromChat({ + request, + context, + token, + }); + + if (!databaseName || !collectionName) { + return await this._askForNamespace({ + command: '/schema', + context, + databaseName, + collectionName, + stream, + }); + } + + if (token.isCancellationRequested) { + return this._handleCancelledRequest({ + context, + stream, + }); + } + + let sampleDocuments: Document[] | undefined; + let amountOfDocumentsSampled: number; + let schema: string | undefined; + try { + ({ + sampleDocuments, + amountOfDocumentsSampled, // There can be fewer than the amount we attempt to sample. + schema, + } = await this._fetchCollectionSchemaAndSampleDocuments({ + databaseName, + schemaFormat: 'full', + collectionName, + amountOfDocumentsToSample: DOCUMENTS_TO_SAMPLE_FOR_SCHEMA_PROMPT, + token, + stream, + })); + + if (!schema || amountOfDocumentsSampled === 0) { + stream.markdown( + vscode.l10n.t( + 'Unable to generate a schema from the collection, no documents found.' + ) + ); + return schemaRequestChatResult(context.history); + } + } catch (e) { + stream.markdown( + vscode.l10n.t( + `Unable to generate a schema from the collection, an error occurred: ${e}` + ) + ); + return schemaRequestChatResult(context.history); + } + + const modelInput = await Prompts.schema.buildMessages({ + request, + context, + databaseName, + amountOfDocumentsSampled, + collectionName, + schema, + connectionNames: this._getConnectionNames(), + ...(sampleDocuments ? { sampleDocuments } : {}), + }); + const response = await this.streamChatResponse({ + modelInput, + stream, + token, + }); + + stream.button({ + command: EXTENSION_COMMANDS.PARTICIPANT_OPEN_RAW_SCHEMA_OUTPUT, + title: vscode.l10n.t('Open JSON Output'), + arguments: [ + { + schema, + } as OpenSchemaCommandArgs, + ], + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'schema', + has_cta: true, + found_namespace: true, + has_runnable_content: false, + output_length: response.outputLength, + }); + + return schemaRequestChatResult(context.history); + } + + // @MongoDB /query find all documents where the "address" has the word Broadway in it. + async handleQueryRequest( + request: vscode.ChatRequest, + context: vscode.ChatContext, + stream: vscode.ChatResponseStream, + token: vscode.CancellationToken + ): Promise { + if (!this._connectionController.getActiveDataService()) { + return this._askToConnect({ + command: '/query', + context, + stream, + }); + } + + if (Prompts.isPromptEmpty(request)) { + if (this._doesLastMessageAskForNamespace(context.history)) { + return this.handleEmptyNamespaceMessage({ + command: '/query', + context, + stream, + }); + } + + stream.markdown(Prompts.query.emptyRequestResponse); + return emptyRequestChatResult(context.history); + } + + // We "prompt chain" to handle the query requests. + // First we ask the model to parse for the database and collection name. + // If they exist, we can then use them in our final completion. + // When they don't exist we ask the user for them. + const { databaseName, collectionName } = await this._getNamespaceFromChat({ + request, + context, + token, + }); + if (!databaseName || !collectionName) { + return await this._askForNamespace({ + command: '/query', + context, + databaseName, + collectionName, + stream, + }); + } + + if (token.isCancellationRequested) { + return this._handleCancelledRequest({ + context, + stream, + }); + } + + let schema: string | undefined; + let sampleDocuments: Document[] | undefined; + try { + ({ schema, sampleDocuments } = + await this._fetchCollectionSchemaAndSampleDocuments({ + databaseName, + collectionName, + token, + stream, + })); + } catch (e) { + // When an error fetching the collection schema or sample docs occurs, + // we still want to continue as it isn't critical, however, + // we do want to notify the user. + stream.markdown( + vscode.l10n.t( + 'An error occurred while fetching the collection schema and sample documents.\nThe generated query will not be able to reference the shape of your data.' + ) + ); + } + + const modelInput = await Prompts.query.buildMessages({ + request, + context, + databaseName, + collectionName, + schema, + connectionNames: this._getConnectionNames(), + ...(sampleDocuments ? { sampleDocuments } : {}), + }); + + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + stream, + token, + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'query', + has_cta: false, + found_namespace: true, + has_runnable_content: hasCodeBlock, + output_length: outputLength, + }); + + return queryRequestChatResult(context.history); + } + + async _handleDocsRequestWithChatbot({ + prompt, + chatId, + token, + stream, + }: { + prompt: string; + chatId: string; + token: vscode.CancellationToken; + stream: vscode.ChatResponseStream; + }): Promise<{ + responseContent: string; + responseReferences?: Reference[]; + docsChatbotMessageId: string; + }> { + stream.push( + new vscode.ChatResponseProgressPart('Consulting MongoDB documentation...') + ); + + let { docsChatbotConversationId } = + this._chatMetadataStore.getChatMetadata(chatId) ?? {}; + const abortController = new AbortController(); + token.onCancellationRequested(() => { + abortController.abort(); + }); + if (!docsChatbotConversationId) { + const conversation = await this._docsChatbotAIService.createConversation({ + signal: abortController.signal, + }); + docsChatbotConversationId = conversation._id; + this._chatMetadataStore.setChatMetadata(chatId, { + docsChatbotConversationId, + }); + log.info('Docs chatbot created for chatId', chatId); + } + + const response = await this._docsChatbotAIService.addMessage({ + message: prompt, + conversationId: docsChatbotConversationId, + signal: abortController.signal, + }); + + log.info('Docs chatbot message sent', { + chatId, + docsChatbotConversationId, + docsChatbotMessageId: response.id, + }); + + return { + responseContent: response.content, + responseReferences: response.references, + docsChatbotMessageId: response.id, + }; + } + + async _handleDocsRequestWithCopilot( + ...args: [ + vscode.ChatRequest, + vscode.ChatContext, + vscode.ChatResponseStream, + vscode.CancellationToken + ] + ): Promise { + const [request, context, stream, token] = args; + const modelInput = await Prompts.generic.buildMessages({ + request, + context, + connectionNames: this._getConnectionNames(), + }); + + const { hasCodeBlock, outputLength } = + await this.streamChatResponseContentWithCodeActions({ + modelInput, + stream, + token, + }); + + this._streamResponseReference({ + reference: { + url: MONGODB_DOCS_LINK, + title: 'View MongoDB documentation', + }, + stream, + }); + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'docs/copilot', + has_cta: true, + found_namespace: false, + has_runnable_content: hasCodeBlock, + output_length: outputLength, + }); + } + + _streamResponseReference({ + reference, + stream, + }: { + reference: Reference; + stream: vscode.ChatResponseStream; + }): void { + const link = new vscode.MarkdownString( + `- [${reference.title}](${reference.url})\n` + ); + link.supportHtml = true; + stream.markdown(link); + } + + async handleDocsRequest( + ...args: [ + vscode.ChatRequest, + vscode.ChatContext, + vscode.ChatResponseStream, + vscode.CancellationToken + ] + ): Promise { + const [request, context, stream, token] = args; + + const chatId = ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ); + let docsResult: { + responseContent?: string; + responseReferences?: Reference[]; + docsChatbotMessageId?: string; + } = {}; + + try { + docsResult = await this._handleDocsRequestWithChatbot({ + prompt: request.prompt, + chatId, + token, + stream, + }); + + if (docsResult.responseReferences) { + for (const reference of docsResult.responseReferences) { + this._streamResponseReference({ + reference, + stream, + }); + } + } + + if (docsResult.responseContent) { + stream.markdown(docsResult.responseContent); + } + + this._telemetryService.trackCopilotParticipantResponse({ + command: 'docs/chatbot', + has_cta: !!docsResult.responseReferences, + found_namespace: false, + has_runnable_content: false, + output_length: docsResult.responseContent?.length ?? 0, + }); + } catch (error) { + // If the docs chatbot API is not available, fall back to Copilot’s LLM and include + // the MongoDB documentation link for users to go to our documentation site directly. + log.error(error); + + if (token.isCancellationRequested) { + return this._handleCancelledRequest({ + context, + stream, + }); + } + + this._telemetryService.track( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED, + { + command: 'docs', + error_name: ParticipantErrorTypes.DOCS_CHATBOT_API, + } + ); + + await this._handleDocsRequestWithCopilot(...args); + } + + return docsRequestChatResult({ + chatId, + docsChatbotMessageId: docsResult.docsChatbotMessageId, + }); + } + + async chatHandler( + ...args: [ + vscode.ChatRequest, + vscode.ChatContext, + vscode.ChatResponseStream, + vscode.CancellationToken + ] + ): Promise { + const [request, , stream] = args; + try { + const hasBeenShownWelcomeMessageAlready = !!this._storageController.get( + StorageVariables.COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE + ); + if (!hasBeenShownWelcomeMessageAlready) { + stream.markdown( + vscode.l10n.t(` +Welcome to MongoDB Participant!\n\n +Interact with your MongoDB clusters and generate MongoDB-related code more efficiently with intelligent AI-powered feature, available today in the MongoDB extension.\n\n +Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more information.\n\n`) + ); + + this._telemetryService.track( + TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN + ); + + await this._storageController.update( + StorageVariables.COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE, + true + ); + } + + switch (request.command) { + case 'query': + return await this.handleQueryRequest(...args); + case 'docs': + return await this.handleDocsRequest(...args); + case 'schema': + return await this.handleSchemaRequest(...args); + default: + if (!request.prompt?.trim()) { + stream.markdown(Prompts.generic.getEmptyRequestResponse()); + return emptyRequestChatResult(args[1].history); + } + + return await this.handleGenericRequest(...args); + } + } catch (e) { + this.handleError(e, request.command || 'generic'); + } + } + + async _rateDocsChatbotMessage( + feedback: vscode.ChatResultFeedback + ): Promise { + const chatId = feedback.result.metadata?.chatId; + if (!chatId) { + return; + } + + const { docsChatbotConversationId } = + this._chatMetadataStore.getChatMetadata(chatId) ?? {}; + if ( + !docsChatbotConversationId || + !feedback.result.metadata?.docsChatbotMessageId + ) { + return; + } + + try { + const rating = await this._docsChatbotAIService.rateMessage({ + conversationId: docsChatbotConversationId, + messageId: feedback.result.metadata?.docsChatbotMessageId, + rating: !!feedback.kind, + }); + log.info('Docs chatbot rating sent', rating); + } catch (error) { + log.error(error); + } + } + + async handleUserFeedback(feedback: vscode.ChatResultFeedback): Promise { + if (feedback.result.metadata?.intent === 'docs') { + await this._rateDocsChatbotMessage(feedback); + } + + // unhelpfulReason is available in insider builds and is accessed through + // https://github.com/microsoft/vscode/blob/main/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts + // Since this is a proposed API, we can't depend on it being available, which is why + // we're dynamically checking for it. + const unhelpfulReason = + 'unhelpfulReason' in feedback + ? (feedback.unhelpfulReason as string) + : undefined; + this._telemetryService.trackCopilotParticipantFeedback({ + feedback: chatResultFeedbackKindToTelemetryValue(feedback.kind), + reason: unhelpfulReason, + response_type: (feedback.result as ChatResult)?.metadata.intent, + }); + } + + _getConnectionNames(): string[] { + return this._connectionController + .getSavedConnections() + .map((connection) => connection.name); + } +} diff --git a/src/participant/prompts/generic.ts b/src/participant/prompts/generic.ts new file mode 100644 index 000000000..2112233da --- /dev/null +++ b/src/participant/prompts/generic.ts @@ -0,0 +1,28 @@ +import * as vscode from 'vscode'; + +import type { PromptArgsBase } from './promptBase'; +import { PromptBase } from './promptBase'; + +import { codeBlockIdentifier } from '../constants'; + +export class GenericPrompt extends PromptBase { + protected getAssistantPrompt(): string { + return `You are a MongoDB expert. +Your task is to help the user with MongoDB related questions. +When applicable, you may suggest MongoDB code, queries, and aggregation pipelines that perform their task. +Rules: +1. Keep your response concise. +2. You should suggest code that is performant and correct. +3. Respond with markdown. +4. When relevant, provide code in a Markdown code block that begins with ${codeBlockIdentifier.start} and ends with ${codeBlockIdentifier.end} +5. Use MongoDB shell syntax for code unless the user requests a specific language. +6. If you require additional information to provide a response, ask the user for it. +7. When specifying a database, use the MongoDB syntax use('databaseName').`; + } + + public getEmptyRequestResponse(): string { + return vscode.l10n.t( + 'Ask anything about MongoDB, from writing queries to questions about your cluster.' + ); + } +} diff --git a/src/participant/prompts/index.ts b/src/participant/prompts/index.ts new file mode 100644 index 000000000..18e4150af --- /dev/null +++ b/src/participant/prompts/index.ts @@ -0,0 +1,37 @@ +import * as vscode from 'vscode'; + +import { GenericPrompt } from './generic'; +import { IntentPrompt } from './intent'; +import { NamespacePrompt } from './namespace'; +import { QueryPrompt } from './query'; +import { SchemaPrompt } from './schema'; + +export class Prompts { + public static generic = new GenericPrompt(); + public static intent = new IntentPrompt(); + public static namespace = new NamespacePrompt(); + public static query = new QueryPrompt(); + public static schema = new SchemaPrompt(); + + public static isPromptEmpty(request: vscode.ChatRequest): boolean { + return !request.prompt || request.prompt.trim().length === 0; + } + + // Check if any of the messages contain user input. + // This is useful since when there's no user input in any + // messages, we can skip some additional processing. + public static doMessagesContainUserInput( + messages: vscode.LanguageModelChatMessage[] + ): boolean { + for (const message of messages) { + if ( + message.role === vscode.LanguageModelChatMessageRole.User && + message.content.trim().length > 0 + ) { + return true; + } + } + + return false; + } +} diff --git a/src/participant/prompts/intent.ts b/src/participant/prompts/intent.ts new file mode 100644 index 000000000..8a1266f69 --- /dev/null +++ b/src/participant/prompts/intent.ts @@ -0,0 +1,55 @@ +import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; +import type { PromptArgsBase } from './promptBase'; +import { PromptBase } from './promptBase'; + +export type PromptIntent = 'Query' | 'Schema' | 'Docs' | 'Default'; + +export class IntentPrompt extends PromptBase { + protected getAssistantPrompt(): string { + return `You are a MongoDB expert. +Your task is to help guide a conversation with a user to the correct handler. +You will be provided a conversation and your task is to determine the intent of the user. +The intent handlers are: +- Query +- Schema +- Docs +- Default +Rules: +1. Respond only with the intent handler. +2. Use the "Query" intent handler when the user is asking for code that relates to a specific collection. +3. Use the "Docs" intent handler when the user is asking a question that involves MongoDB documentation. +4. Use the "Schema" intent handler when the user is asking for the schema or shape of documents of a specific collection. +5. Use the "Default" intent handler when a user is asking for code that does NOT relate to a specific collection. +6. Use the "Default" intent handler for everything that may not be handled by another handler. +7. If you are uncertain of the intent, use the "Default" intent handler. + +Example: +User: How do I create an index in my pineapples collection? +Response: +Query + +Example: +User: +What is $vectorSearch? +Response: +Docs`; + } + + getIntentFromModelResponse(response: string): PromptIntent { + response = response.trim(); + switch (response) { + case 'Query': + return 'Query'; + case 'Schema': + return 'Schema'; + case 'Docs': + return 'Docs'; + default: + return 'Default'; + } + } + + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return 'intent'; + } +} diff --git a/src/participant/prompts/namespace.ts b/src/participant/prompts/namespace.ts new file mode 100644 index 000000000..c5428f191 --- /dev/null +++ b/src/participant/prompts/namespace.ts @@ -0,0 +1,58 @@ +import type { InternalPromptPurpose } from '../../telemetry/telemetryService'; +import type { PromptArgsBase } from './promptBase'; +import { PromptBase } from './promptBase'; + +const DB_NAME_ID = 'DATABASE_NAME'; +const COL_NAME_ID = 'COLLECTION_NAME'; + +const DB_NAME_REGEX = `${DB_NAME_ID}: (.*)`; +const COL_NAME_REGEX = `${COL_NAME_ID}: (.*)`; + +export class NamespacePrompt extends PromptBase { + protected getAssistantPrompt(): string { + return `You are a MongoDB expert. +Parse all user messages to find a database name and a collection name. +Respond in the format: +${DB_NAME_ID}: X +${COL_NAME_ID}: Y +where X and Y are the respective names. +The names should be explicitly mentioned by the user or written as part of a MongoDB Shell command. +If you cannot find the names do not imagine names. +If only one of the names is found, respond only with the found name. +Your response must be concise and correct. + +When no names are found, respond with: +No names found. + +___ +Example 1: +User: How many documents are in the sightings collection in the ufo database? +Response: +${DB_NAME_ID}: ufo +${COL_NAME_ID}: sightings +___ +Example 2: +User: How do I create an index in my pineapples collection? +Response: +${COL_NAME_ID}: pineapples +___ +Example 3: +User: Where is the best hummus in Berlin? +Response: +No names found. +`; + } + + extractDatabaseAndCollectionNameFromResponse(text: string): { + databaseName?: string; + collectionName?: string; + } { + const databaseName = text.match(DB_NAME_REGEX)?.[1].trim(); + const collectionName = text.match(COL_NAME_REGEX)?.[1].trim(); + return { databaseName, collectionName }; + } + + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return 'namespace'; + } +} diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts new file mode 100644 index 000000000..949b4f3d0 --- /dev/null +++ b/src/participant/prompts/promptBase.ts @@ -0,0 +1,172 @@ +import * as vscode from 'vscode'; +import type { ChatResult, ParticipantResponseType } from '../constants'; +import type { + InternalPromptPurpose, + ParticipantPromptProperties, +} from '../../telemetry/telemetryService'; + +export interface PromptArgsBase { + request: { + prompt: string; + command?: string; + }; + context: vscode.ChatContext; + connectionNames: string[]; +} + +export interface UserPromptResponse { + prompt: string; + hasSampleDocs: boolean; +} + +export interface ModelInput { + messages: vscode.LanguageModelChatMessage[]; + stats: ParticipantPromptProperties; +} + +export abstract class PromptBase { + protected abstract getAssistantPrompt(args: TArgs): string; + + protected get internalPurposeForTelemetry(): InternalPromptPurpose { + return undefined; + } + + protected getUserPrompt(args: TArgs): Promise { + return Promise.resolve({ + prompt: args.request.prompt, + hasSampleDocs: false, + }); + } + + async buildMessages(args: TArgs): Promise { + let historyMessages = this.getHistoryMessages(args); + // If the current user's prompt is a connection name, and the last + // message was to connect. We want to use the last + // message they sent before the connection name as their prompt. + if (args.connectionNames.includes(args.request.prompt)) { + const history = args.context.history; + const previousResponse = history[ + history.length - 1 + ] as vscode.ChatResponseTurn; + const intent = (previousResponse?.result as ChatResult)?.metadata.intent; + if (intent === 'askToConnect') { + // Go through the history in reverse order to find the last user message. + for (let i = history.length - 1; i >= 0; i--) { + if (history[i] instanceof vscode.ChatRequestTurn) { + // Rewrite the arguments so that the prompt is the last user message from history + args = { + ...args, + request: { + ...args.request, + prompt: (history[i] as vscode.ChatRequestTurn).prompt, + }, + }; + + // Remove the item from the history messages array. + historyMessages = historyMessages.slice(0, i); + break; + } + } + } + } + + const { prompt, hasSampleDocs } = await this.getUserPrompt(args); + const messages = [ + // eslint-disable-next-line new-cap + vscode.LanguageModelChatMessage.Assistant(this.getAssistantPrompt(args)), + ...historyMessages, + // eslint-disable-next-line new-cap + vscode.LanguageModelChatMessage.User(prompt), + ]; + + return { + messages, + stats: this.getStats(messages, args, hasSampleDocs), + }; + } + + protected getStats( + messages: vscode.LanguageModelChatMessage[], + { request, context }: TArgs, + hasSampleDocs: boolean + ): ParticipantPromptProperties { + return { + total_message_length: messages.reduce( + (acc, message) => acc + message.content.length, + 0 + ), + user_input_length: request.prompt.length, + has_sample_documents: hasSampleDocs, + command: request.command || 'generic', + history_size: context.history.length, + internal_purpose: this.internalPurposeForTelemetry, + }; + } + + // When passing the history to the model we only want contextual messages + // to be passed. This function parses through the history and returns + // the messages that are valuable to keep. + // eslint-disable-next-line complexity + protected getHistoryMessages({ + connectionNames, + context, + }: { + connectionNames: string[]; // Used to scrape the connecting messages from the history. + context: vscode.ChatContext; + }): vscode.LanguageModelChatMessage[] { + const messages: vscode.LanguageModelChatMessage[] = []; + + for (const historyItem of context.history) { + if (historyItem instanceof vscode.ChatRequestTurn) { + if ( + historyItem.prompt?.trim().length === 0 || + connectionNames?.includes(historyItem.prompt) + ) { + // When the message is empty or a connection name then we skip it. + // It's probably going to be the response to the connect step. + continue; + } + + // eslint-disable-next-line new-cap + messages.push(vscode.LanguageModelChatMessage.User(historyItem.prompt)); + } + + if (historyItem instanceof vscode.ChatResponseTurn) { + let message = ''; + + // Skip a response to an empty user prompt message or connect message. + const responseTypesToSkip: ParticipantResponseType[] = [ + 'emptyRequest', + 'askToConnect', + ]; + if ( + responseTypesToSkip.indexOf( + (historyItem.result as ChatResult)?.metadata?.intent + ) > -1 + ) { + continue; + } + + for (const fragment of historyItem.response) { + if (fragment instanceof vscode.ChatResponseMarkdownPart) { + message += fragment.value.value; + + if ( + (historyItem.result as ChatResult)?.metadata?.intent === + 'askForNamespace' + ) { + // When the message is the assistant asking for part of a namespace, + // we only want to include the question asked, not the user's + // database and collection names in the history item. + break; + } + } + } + // eslint-disable-next-line new-cap + messages.push(vscode.LanguageModelChatMessage.Assistant(message)); + } + } + + return messages; + } +} diff --git a/src/participant/prompts/query.ts b/src/participant/prompts/query.ts new file mode 100644 index 000000000..1efef4ba6 --- /dev/null +++ b/src/participant/prompts/query.ts @@ -0,0 +1,86 @@ +import * as vscode from 'vscode'; +import type { Document } from 'bson'; + +import { getStringifiedSampleDocuments } from '../sampleDocuments'; +import type { PromptArgsBase, UserPromptResponse } from './promptBase'; +import { codeBlockIdentifier } from '../constants'; +import { PromptBase } from './promptBase'; + +interface QueryPromptArgs extends PromptArgsBase { + databaseName: string; + collectionName: string; + schema?: string; + sampleDocuments?: Document[]; + connectionNames: string[]; +} + +export class QueryPrompt extends PromptBase { + protected getAssistantPrompt(): string { + return `You are a MongoDB expert. +Your task is to help the user craft MongoDB shell syntax code to perform their task. +Keep your response concise. +You must suggest code that is performant and correct. +Respond with markdown, write code in a Markdown code block that begins with ${codeBlockIdentifier.start} and ends with ${codeBlockIdentifier.end}. +Respond in MongoDB shell syntax using the ${codeBlockIdentifier.start} code block syntax. + +Concisely explain the code snippet you have generated. + +Example 1: +User: Documents in the orders db, sales collection, where the date is in 2014 and group the total sales for each product. +Response: +${codeBlockIdentifier.start} +use('orders'); +db.getCollection('sales').aggregate([ + // Find all of the sales that occurred in 2014. + { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } }, + // Group the total sales for each product. + { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: [ '$price', '$quantity' ] } } } } +]); +${codeBlockIdentifier.end} + +Example 2: +User: How do I create an index on the name field in my users collection?. +Response: +${codeBlockIdentifier.start} +use('test'); +db.getCollection('users').createIndex({ name: 1 }); +${codeBlockIdentifier.end} + +MongoDB command to specify database: +use(''); + +MongoDB command to specify collection: +db.getCollection('');\n`; + } + + async getUserPrompt({ + databaseName = 'mongodbVSCodeCopilotDB', + collectionName = 'test', + request, + schema, + sampleDocuments, + }: QueryPromptArgs): Promise { + let prompt = request.prompt; + prompt += `\nDatabase name: ${databaseName}\n`; + prompt += `Collection name: ${collectionName}\n`; + if (schema) { + prompt += `Collection schema: ${schema}\n`; + } + + const sampleDocumentsPrompt = await getStringifiedSampleDocuments({ + sampleDocuments, + prompt, + }); + + return { + prompt: `${prompt}${sampleDocumentsPrompt}`, + hasSampleDocs: !!sampleDocumentsPrompt, + }; + } + + get emptyRequestResponse(): string { + return vscode.l10n.t( + 'Please specify a question when using this command. Usage: @MongoDB /query find documents where "name" contains "database".' + ); + } +} diff --git a/src/participant/prompts/schema.ts b/src/participant/prompts/schema.ts new file mode 100644 index 000000000..ca8b54b26 --- /dev/null +++ b/src/participant/prompts/schema.ts @@ -0,0 +1,45 @@ +import type { UserPromptResponse } from './promptBase'; +import { PromptBase, type PromptArgsBase } from './promptBase'; + +export const DOCUMENTS_TO_SAMPLE_FOR_SCHEMA_PROMPT = 100; + +export type OpenSchemaCommandArgs = { + schema: string; +}; + +export interface SchemaPromptArgs extends PromptArgsBase { + databaseName: string; + collectionName: string; + schema: string; + amountOfDocumentsSampled: number; +} + +export class SchemaPrompt extends PromptBase { + getAssistantPrompt({ amountOfDocumentsSampled }: SchemaPromptArgs): string { + return `You are a senior engineer who describes the schema of documents in a MongoDB database. +The schema is generated from a sample of documents in the user's collection. +You must follow these rules. +Rule 1: Try to be as concise as possible. +Rule 2: Pay attention to the JSON schema. +Rule 3: Mention the amount of documents sampled in your response. +Amount of documents sampled: ${amountOfDocumentsSampled}.`; + } + + getUserPrompt({ + databaseName, + collectionName, + request, + schema, + }: SchemaPromptArgs): Promise { + const prompt = request.prompt; + return Promise.resolve({ + prompt: `${ + prompt ? `The user provided additional information: "${prompt}"\n` : '' + }Database name: ${databaseName} +Collection name: ${collectionName} +Schema: +${schema}`, + hasSampleDocs: false, + }); + } +} diff --git a/src/participant/sampleDocuments.ts b/src/participant/sampleDocuments.ts new file mode 100644 index 000000000..4945839c0 --- /dev/null +++ b/src/participant/sampleDocuments.ts @@ -0,0 +1,74 @@ +import { toJSString } from 'mongodb-query-parser'; +import type { Document } from 'bson'; +import { getCopilotModel } from './model'; + +const MAX_ARRAY_LENGTH_OF_SAMPLE_DOCUMENT_VALUE = 3; + +const MAX_STRING_LENGTH_OF_SAMPLE_DOCUMENT_VALUE = 20; + +export function getSimplifiedSampleDocuments(obj: Document[]): Document[] { + function truncate(value: any): any { + if (typeof value === 'string') { + return value.slice(0, MAX_STRING_LENGTH_OF_SAMPLE_DOCUMENT_VALUE); + } else if (typeof value === 'object' && value !== null) { + if (Array.isArray(value)) { + value = value.slice(0, MAX_ARRAY_LENGTH_OF_SAMPLE_DOCUMENT_VALUE); + } + // Recursively truncate strings in nested objects or arrays. + for (const key in value) { + if (value.hasOwnProperty(key)) { + value[key] = truncate(value[key]); + } + } + } + return value; + } + + return truncate(obj); +} + +export async function getStringifiedSampleDocuments({ + prompt, + sampleDocuments, +}: { + prompt: string; + sampleDocuments?: Document[]; +}): Promise { + if (!sampleDocuments?.length) { + return ''; + } + + const model = await getCopilotModel(); + if (!model) { + return ''; + } + + let additionToPrompt: Document[] | Document = sampleDocuments; + let promptInputTokens = + (await model.countTokens(prompt + toJSString(sampleDocuments))) || 0; + + // First check the length of all stringified sample documents. + // If the resulting prompt is too large, proceed with only 1 sample document. + // We also convert an array that contains only 1 element to a single document. + if ( + promptInputTokens > model.maxInputTokens || + sampleDocuments.length === 1 + ) { + additionToPrompt = sampleDocuments[0]; + } + + const stringifiedDocuments = toJSString(additionToPrompt); + + // TODO: model.countTokens will sometimes return undefined - at least in tests. We should investigate why. + promptInputTokens = + (await model.countTokens(prompt + stringifiedDocuments)) || 0; + + // Add sample documents to the prompt only when it fits in the context window. + if (promptInputTokens <= model.maxInputTokens) { + return `\nSample document${ + Array.isArray(additionToPrompt) ? 's' : '' + }: ${stringifiedDocuments}\n`; + } + + return ''; +} diff --git a/src/participant/schema.ts b/src/participant/schema.ts new file mode 100644 index 000000000..ad247eb70 --- /dev/null +++ b/src/participant/schema.ts @@ -0,0 +1,102 @@ +import type { + SimplifiedSchema, + SimplifiedSchemaArrayType, + SimplifiedSchemaDocumentType, + SimplifiedSchemaType, +} from 'mongodb-schema'; + +const PROPERTY_REGEX = '^[a-zA-Z_$][0-9a-zA-Z_$]*$'; + +export class SchemaFormatter { + static getSchemaFromTypes(pInput: SimplifiedSchema): string { + return new SchemaFormatter().format(pInput); + } + + schemaString = ''; + + format(pInitial: SimplifiedSchema): string { + this.processDocumentType('', pInitial); + return this.schemaString; + } + + private processSchemaTypeList( + prefix: string, + pTypes: SimplifiedSchemaType[] + ): void { + if (pTypes.length !== 0) { + this.processSchemaType(prefix, pTypes[0]); + } + } + + private processSchemaType(prefix: string, pType: SimplifiedSchemaType): void { + const bsonType = pType.bsonType; + if (bsonType === 'Document') { + const fields = (pType as SimplifiedSchemaDocumentType).fields; + + if (Object.keys(fields).length === 0) { + this.addToFormattedSchemaString(prefix + ': Document'); + return; + } + + this.processDocumentType(prefix, fields); + return; + } + + if (bsonType === 'Array') { + const types = (pType as SimplifiedSchemaArrayType).types; + + if (types.length === 0) { + this.addToFormattedSchemaString(prefix + ': ' + 'Array'); + return; + } + + const firstType = types[0].bsonType; + if (firstType !== 'Array' && firstType !== 'Document') { + this.addToFormattedSchemaString( + prefix + ': ' + 'Array<' + firstType + '>' + ); + return; + } + + // Array of documents or arrays. + // We only use the first type. + this.processSchemaType(prefix + '[]', types[0]); + return; + } + + this.addToFormattedSchemaString(prefix + ': ' + bsonType); + } + + private processDocumentType(prefix: string, pDoc: SimplifiedSchema): void { + if (!pDoc) { + return; + } + + Object.keys(pDoc).forEach((key) => { + const keyAsString = this.getPropAsString(key); + this.processSchemaTypeList( + prefix + (prefix.length === 0 ? '' : '.') + keyAsString, + pDoc[key]?.types + ); + }); + } + + getPropAsString(pProp: string): string { + if (pProp.match(PROPERTY_REGEX)) { + return pProp; + } + + try { + return JSON.stringify(pProp); + } catch (e) { + return pProp; + } + } + + addToFormattedSchemaString(fieldAndType: string): void { + if (this.schemaString.length > 0) { + this.schemaString += '\n'; + } + this.schemaString += fieldAndType; + } +} diff --git a/src/participant/streamParsing.ts b/src/participant/streamParsing.ts new file mode 100644 index 000000000..93bb5dad9 --- /dev/null +++ b/src/participant/streamParsing.ts @@ -0,0 +1,95 @@ +function escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * This function, provided a stream of text fragments, will stream the + * content to the provided stream and call the onStreamIdentifier function + * when an identifier is streamed. This is useful for inserting code actions + * into a chat response, whenever a code block has been written. + */ +export async function processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier, +}: { + processStreamFragment: (fragment: string) => void; + onStreamIdentifier: (content: string) => void; + inputIterable: AsyncIterable; + identifier: { + start: string; + end: string; + }; +}): Promise { + const escapedIdentifierStart = escapeRegex(identifier.start); + const escapedIdentifierEnd = escapeRegex(identifier.end); + const regex = new RegExp( + `${escapedIdentifierStart}([\\s\\S]*?)${escapedIdentifierEnd}`, + 'g' + ); + + let contentSinceLastIdentifier = ''; + for await (const fragment of inputIterable) { + contentSinceLastIdentifier += fragment; + + let lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = regex.exec(contentSinceLastIdentifier)) !== null) { + const endIndex = regex.lastIndex; + + // Stream content up to the end of the identifier. + const contentToStream = contentSinceLastIdentifier.slice( + lastIndex, + endIndex + ); + processStreamFragment(contentToStream); + + const identifierContent = match[1]; + onStreamIdentifier(identifierContent); + + lastIndex = endIndex; + } + + if (lastIndex > 0) { + // Remove all of the processed content. + contentSinceLastIdentifier = contentSinceLastIdentifier.slice(lastIndex); + // Reset the regex. + regex.lastIndex = 0; + } else { + // Clear as much of the content as we can safely. + const maxUnprocessedLength = identifier.start.length - 1; + if (contentSinceLastIdentifier.length > maxUnprocessedLength) { + const identifierIndex = contentSinceLastIdentifier.indexOf( + identifier.start + ); + if (identifierIndex > -1) { + // We have an identifier, so clear up until the identifier. + const contentToStream = contentSinceLastIdentifier.slice( + 0, + identifierIndex + ); + processStreamFragment(contentToStream); + contentSinceLastIdentifier = + contentSinceLastIdentifier.slice(identifierIndex); + } else { + // No identifier, so clear up until the last maxUnprocessedLength. + const processUpTo = + contentSinceLastIdentifier.length - maxUnprocessedLength; + const contentToStream = contentSinceLastIdentifier.slice( + 0, + processUpTo + ); + processStreamFragment(contentToStream); + contentSinceLastIdentifier = + contentSinceLastIdentifier.slice(processUpTo); + } + } + } + } + + // Finish up anything not streamed yet. + if (contentSinceLastIdentifier.length > 0) { + processStreamFragment(contentSinceLastIdentifier); + } +} diff --git a/src/storage/connectionStorage.ts b/src/storage/connectionStorage.ts index 56bb2f166..e233ad4d5 100644 --- a/src/storage/connectionStorage.ts +++ b/src/storage/connectionStorage.ts @@ -23,6 +23,7 @@ export interface StoreConnectionInfo { storageLocation: StorageLocation; secretStorageLocation?: SecretStorageLocationType; connectionOptions?: ConnectionOptions; + lastUsed?: Date; // Date and time when the connection was last used, i.e. connected with. } type StoreConnectionInfoWithConnectionOptions = StoreConnectionInfo & diff --git a/src/storage/storageController.ts b/src/storage/storageController.ts index 0148208b9..e19eae149 100644 --- a/src/storage/storageController.ts +++ b/src/storage/storageController.ts @@ -13,6 +13,7 @@ export enum StorageVariables { GLOBAL_ANONYMOUS_ID = 'GLOBAL_ANONYMOUS_ID', // Only exists on workspaceState. WORKSPACE_SAVED_CONNECTIONS = 'WORKSPACE_SAVED_CONNECTIONS', + COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE = 'COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE', } // Typically variables default to 'GLOBAL' scope. @@ -54,6 +55,7 @@ interface StorageVariableContents { [StorageVariables.GLOBAL_SURVEY_SHOWN]: string; [StorageVariables.GLOBAL_SAVED_CONNECTIONS]: ConnectionsFromStorage; [StorageVariables.WORKSPACE_SAVED_CONNECTIONS]: ConnectionsFromStorage; + [StorageVariables.COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE]: boolean; } type StoredVariableName = keyof StorageVariableContents; type StoredItem = StorageVariableContents[T]; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index f3bcb1bf4..93220661e 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -12,6 +12,7 @@ import { getConnectionTelemetryProperties } from './connectionTelemetry'; import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; +import type { ParticipantResponseType } from '../participant/constants'; const log = createLogger('telemetry'); // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -93,7 +94,53 @@ type SavedConnectionsLoadedProperties = { connections_with_secrets_in_secret_storage: number; }; -export type TelemetryEventProperties = +type TelemetryFeedbackKind = 'positive' | 'negative' | undefined; + +type ParticipantFeedbackProperties = { + feedback: TelemetryFeedbackKind; + response_type: ParticipantResponseType; + reason?: String; +}; + +type ParticipantResponseFailedProperties = { + command: string; + error_code?: string; + error_name: ParticipantErrorTypes; +}; + +export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; + +export type ParticipantPromptProperties = { + command: string; + user_input_length: number; + total_message_length: number; + has_sample_documents: boolean; + history_size: number; + internal_purpose: InternalPromptPurpose; +}; + +export type ParticipantResponseProperties = { + command: string; + has_cta: boolean; + has_runnable_content: boolean; + found_namespace: boolean; + output_length: number; +}; + +export function chatResultFeedbackKindToTelemetryValue( + kind: vscode.ChatResultFeedbackKind +): TelemetryFeedbackKind { + switch (kind) { + case vscode.ChatResultFeedbackKind.Helpful: + return 'positive'; + case vscode.ChatResultFeedbackKind.Unhelpful: + return 'negative'; + default: + return undefined; + } +} + +type TelemetryEventProperties = | PlaygroundTelemetryEventProperties | LinkClickedTelemetryEventProperties | ExtensionCommandRunTelemetryEventProperties @@ -107,7 +154,11 @@ export type TelemetryEventProperties = | PlaygroundLoadedTelemetryEventProperties | KeytarSecretsMigrationFailedProperties | SavedConnectionsLoadedProperties - | SurveyActionProperties; + | SurveyActionProperties + | ParticipantFeedbackProperties + | ParticipantResponseFailedProperties + | ParticipantPromptProperties + | ParticipantResponseProperties; export enum TelemetryEventTypes { PLAYGROUND_CODE_EXECUTED = 'Playground Code Executed', @@ -127,6 +178,19 @@ export enum TelemetryEventTypes { SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded', SURVEY_CLICKED = 'Survey link clicked', SURVEY_DISMISSED = 'Survey prompt dismissed', + PARTICIPANT_FEEDBACK = 'Participant Feedback', + PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', + PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', + PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', + PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', +} + +export enum ParticipantErrorTypes { + CHAT_MODEL_OFF_TOPIC = 'Chat Model Off Topic', + INVALID_PROMPT = 'Invalid Prompt', + FILTERED = 'Filtered by Responsible AI Service', + OTHER = 'Other', + DOCS_CHATBOT_API = 'Docs Chatbot API Issue', } /** @@ -162,13 +226,12 @@ export default class TelemetryService { ); // eslint-disable-next-line no-sync const constantsFile = fs.readFileSync(segmentKeyFileLocation, 'utf8'); - const constants = JSON.parse(constantsFile) as { segmentKey: string }; - - log.info('SegmentKey was found', { type: typeof constants.segmentKey }); - - return constants.segmentKey; + const { segmentKey } = JSON.parse(constantsFile) as { + segmentKey?: string; + }; + return segmentKey; } catch (error) { - log.error('SegmentKey was not found', error); + log.error('Failed to read segmentKey from the constants file', error); return; } } @@ -215,7 +278,7 @@ export default class TelemetryService { return true; } - _segmentAnalyticsTrack(segmentProperties: SegmentProperties) { + _segmentAnalyticsTrack(segmentProperties: SegmentProperties): void { if (!this._isTelemetryFeatureEnabled()) { return; } @@ -250,7 +313,7 @@ export default class TelemetryService { async _getConnectionTelemetryProperties( dataService: DataService, connectionType: ConnectionTypes - ) { + ): Promise { return await getConnectionTelemetryProperties(dataService, connectionType); } @@ -298,7 +361,7 @@ export default class TelemetryService { return 'other'; } - getTelemetryUserIdentity() { + getTelemetryUserIdentity(): { anonymousId: string } { return { anonymousId: this._segmentAnonymousId, }; @@ -363,7 +426,7 @@ export default class TelemetryService { trackSavedConnectionsLoaded( savedConnectionsLoadedProps: SavedConnectionsLoadedProperties - ) { + ): void { this.track( TelemetryEventTypes.SAVED_CONNECTIONS_LOADED, savedConnectionsLoadedProps @@ -372,10 +435,22 @@ export default class TelemetryService { trackKeytarSecretsMigrationFailed( keytarSecretsMigrationFailedProps: KeytarSecretsMigrationFailedProperties - ) { + ): void { this.track( TelemetryEventTypes.KEYTAR_SECRETS_MIGRATION_FAILED, keytarSecretsMigrationFailedProps ); } + + trackCopilotParticipantFeedback(props: ParticipantFeedbackProperties): void { + this.track(TelemetryEventTypes.PARTICIPANT_FEEDBACK, props); + } + + trackCopilotParticipantPrompt(stats: ParticipantPromptProperties): void { + this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats); + } + + trackCopilotParticipantResponse(props: ParticipantResponseProperties): void { + this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props); + } } diff --git a/src/templates/playgroundBasicTextTemplate.ts b/src/templates/playgroundBasicTextTemplate.ts new file mode 100644 index 000000000..6114f5840 --- /dev/null +++ b/src/templates/playgroundBasicTextTemplate.ts @@ -0,0 +1,14 @@ +const template = `/* global use, db */ +// MongoDB Playground +// To disable this template go to Settings | MongoDB | Use Default Template For Playground. +// Make sure you are connected to enable completions and to be able to run a playground. +// Use Ctrl+Space inside a snippet or a string literal to trigger completions. +// The result of the last command run in a playground is shown on the results panel. +// By default the first 20 documents will be returned with a cursor. +// Use 'console.log()' to print to the debug output. +// For more documentation on playgrounds please refer to +// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/ +PLAYGROUND_CONTENT +`; + +export default template; diff --git a/src/test/ai-accuracy-tests/ai-accuracy-tests.ts b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts new file mode 100644 index 000000000..1b3d45636 --- /dev/null +++ b/src/test/ai-accuracy-tests/ai-accuracy-tests.ts @@ -0,0 +1,776 @@ +/* eslint-disable no-console */ +import { expect } from 'chai'; +import { MongoClient } from 'mongodb'; +import { execFile as callbackExecFile } from 'child_process'; +import { MongoCluster } from 'mongodb-runner'; +import path from 'path'; +import util from 'util'; +import os from 'os'; +import * as vscode from 'vscode'; + +import { loadFixturesToDB, reloadFixture } from './fixtures/fixture-loader'; +import type { Fixtures } from './fixtures/fixture-loader'; +import { AIBackend } from './ai-backend'; +import type { ChatCompletion } from './ai-backend'; +import { + createTestResultsHTMLPage, + type TestOutputs, + type TestResult, +} from './create-test-results-html-page'; +import { anyOf, runCodeInMessage } from './assertions'; +import { Prompts } from '../../participant/prompts'; +import type { ModelInput } from '../../participant/prompts/promptBase'; + +const numberOfRunsPerTest = 1; + +// When true, we will log the entire prompt we send to the model for each test. +const DEBUG_PROMPTS = process.env.DEBUG_PROMPTS === 'true'; + +type AssertProps = { + responseContent: string; + connectionString: string; + fixtures: Fixtures; + mongoClient: MongoClient; +}; + +type TestCase = { + testCase: string; + type: 'intent' | 'generic' | 'query' | 'namespace'; + userInput: string; + // Some tests can edit the documents in a collection. + // As we want tests to run in isolation this flag will cause the fixture + // to be reloaded on each run of the tests so subsequent tests are not impacted. + reloadFixtureOnEachRun?: boolean; + databaseName?: string; + collectionName?: string; + includeSampleDocuments?: boolean; + accuracyThresholdOverride?: number; + assertResult: (props: AssertProps) => Promise | void; + only?: boolean; // Translates to mocha's it.only so only this test will run. +}; + +const namespaceTestCases: (TestCase & { + type: 'namespace'; +})[] = [ + { + testCase: 'Namespace included in query', + type: 'namespace', + userInput: + 'How many documents are in the tempReadings collection in the pools database?', + assertResult: ({ responseContent }: AssertProps): void => { + const namespace = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse( + responseContent + ); + + expect(namespace.databaseName).to.equal('pools'); + expect(namespace.collectionName).to.equal('tempReadings'); + }, + }, + { + testCase: 'No namespace included in basic query', + type: 'namespace', + userInput: 'How many documents are in the collection?', + assertResult: ({ responseContent }: AssertProps): void => { + const namespace = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse( + responseContent + ); + + expect(namespace.databaseName).to.equal(undefined); + expect(namespace.collectionName).to.equal(undefined); + }, + }, + { + testCase: 'Only collection mentioned in query', + type: 'namespace', + userInput: + 'How do I create a new user with read write permissions on the orders collection?', + assertResult: ({ responseContent }: AssertProps): void => { + const namespace = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse( + responseContent + ); + + expect(namespace.databaseName).to.equal(undefined); + expect(namespace.collectionName).to.equal('orders'); + }, + }, + { + testCase: 'Only database mentioned in query', + type: 'namespace', + userInput: + 'How do I create a new user with read write permissions on the orders db?', + assertResult: ({ responseContent }: AssertProps): void => { + const namespace = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse( + responseContent + ); + + expect(namespace.databaseName).to.equal('orders'); + expect(namespace.collectionName).to.equal(undefined); + }, + }, +]; + +const queryTestCases: (TestCase & { + type: 'query'; +})[] = [ + { + testCase: 'Basic query', + type: 'query', + databaseName: 'UFO', + collectionName: 'sightings', + userInput: 'How many documents are in the collection?', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const result = await runCodeInMessage(responseContent, connectionString); + + const totalResponse = `${result.printOutput.join('')}${ + result.data?.result?.content + }`; + + const number = totalResponse.match(/\d+/); + expect(number?.[0]).to.equal('5'); + }, + }, + { + testCase: 'Slightly complex updateOne', + type: 'query', + databaseName: 'CookBook', + collectionName: 'recipes', + reloadFixtureOnEachRun: true, + userInput: + "Update the Beef Wellington recipe to have its preparation time 150 minutes and set the difficulty level to 'Very Hard'", + assertResult: async ({ + responseContent, + connectionString, + mongoClient, + fixtures, + }: AssertProps): Promise => { + const documentsBefore = await mongoClient + .db('CookBook') + .collection('recipes') + .find() + .toArray(); + expect(documentsBefore).to.deep.equal( + fixtures.CookBook.recipes.documents + ); + + await runCodeInMessage(responseContent, connectionString); + const documents = await mongoClient + .db('CookBook') + .collection('recipes') + .find() + .toArray(); + + expect(documents).to.deep.equal( + fixtures.CookBook.recipes.documents.map((doc) => { + if (doc.title === 'Beef Wellington') { + return { + ...doc, + preparationTime: 150, + difficulty: 'Very Hard', + }; + } + return doc; + }) + ); + }, + }, + { + testCase: 'Aggregation with averaging and filtering', + type: 'query', + databaseName: 'pets', + collectionName: 'competition-results', + userInput: + 'What is the average score for dogs competing in the best costume category? Put it in a field called "avgScore"', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + + expect(output.data?.result?.content[0]).to.deep.equal({ + avgScore: 9.3, + }); + }, + }, + { + testCase: 'Create an index', + type: 'query', + databaseName: 'FarmData', + collectionName: 'Pineapples', + reloadFixtureOnEachRun: true, + userInput: + 'How to index the harvested date and sweetness to speed up requests for sweet pineapples harvested after a specific date?', + assertResult: async ({ + responseContent, + connectionString, + mongoClient, + }: AssertProps): Promise => { + const indexesBefore = await mongoClient + .db('FarmData') + .collection('Pineapples') + .listIndexes() + .toArray(); + expect(indexesBefore.length).to.equal(1); + await runCodeInMessage(responseContent, connectionString); + + const indexes = await mongoClient + .db('FarmData') + .collection('Pineapples') + .listIndexes() + .toArray(); + + expect(indexes.length).to.equal(2); + expect( + indexes.filter((index) => index.name !== '_id_')[0]?.key + ).to.have.keys(['harvestedDate', 'sweetnessScale']); + }, + }, + { + testCase: 'Aggregation with an or or $in, with sample docs', + type: 'query', + databaseName: 'Antiques', + collectionName: 'items', + includeSampleDocuments: true, + userInput: + 'which collectors specialize only in mint items? and are located in London or New York? an array of their names in a field called collectors', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + + expect(output.data?.result?.content?.[0].collectors).to.have.lengthOf(2); + expect(output.data?.result?.content[0].collectors).to.include('John Doe'); + expect(output.data?.result?.content[0].collectors).to.include('Monkey'); + }, + }, + { + testCase: 'Complex aggregation with string and number manipulation', + type: 'query', + databaseName: 'CookBook', + collectionName: 'recipes', + userInput: + 'what percentage of recipes have "salt" in their ingredients? "ingredients" is a field ' + + 'with an array of strings of the ingredients. Only consider recipes ' + + 'that have the "difficulty Medium or Easy. Return is as a string named "saltPercentage" like ' + + '"75%", rounded to the nearest whole number.', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + + anyOf([ + (): void => { + const lines = responseContent.trim().split('\n'); + const lastLine = lines[lines.length - 1]; + + expect(lastLine).to.include('saltPercentage'); + expect(output.data?.result?.content).to.include('67%'); + }, + (): void => { + expect(output.printOutput[output.printOutput.length - 1]).to.equal( + "{ saltPercentage: '67%' }" + ); + }, + (): void => { + expect(output.data?.result?.content[0].saltPercentage).to.equal( + '67%' + ); + }, + ])(null); + }, + }, +]; + +const intentTestCases: (TestCase & { + type: 'intent'; +})[] = [ + { + testCase: 'Docs intent', + type: 'intent', + userInput: + 'Where can I find more information on how to connect to MongoDB?', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Docs'); + }, + }, + { + testCase: 'Docs intent 2', + type: 'intent', + userInput: 'What are the options when creating an aggregation cursor?', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Docs'); + }, + }, + { + testCase: 'Query intent', + type: 'intent', + userInput: + 'which collectors specialize only in mint items? and are located in London or New York? an array of their names in a field called collectors', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Query'); + }, + }, + { + testCase: 'Schema intent', + type: 'intent', + userInput: 'What do the documents in the collection pineapple look like?', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Schema'); + }, + }, + { + testCase: 'Default/Generic intent 1', + type: 'intent', + userInput: 'How can I connect to MongoDB?', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Default'); + }, + }, + { + testCase: 'Default/Generic intent 2', + type: 'intent', + userInput: 'What is the size breakdown of all of the databases?', + assertResult: ({ responseContent }: AssertProps): void => { + expect(responseContent).to.equal('Default'); + }, + }, +]; + +const genericTestCases: (TestCase & { + type: 'generic'; +})[] = [ + { + testCase: 'Database meta data question', + type: 'generic', + userInput: + 'How do I print the name and size of the largest database? Using the print function', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + const printOutput = output.printOutput.join(''); + + // Don't check the name since they're all the base 8192. + expect(printOutput).to.include('8192'); + }, + }, + { + testCase: 'Code question with database, collection, and fields named', + type: 'generic', + userInput: + 'How many sightings happened in the "year" "2020" and "2021"? database "UFO" collection "sightings". code to just return the one total number. also, the year is a string', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + anyOf([ + (): void => { + expect(output.printOutput.join('')).to.equal('2'); + }, + (): void => { + expect(output.data?.result?.content).to.equal('2'); + }, + (): void => { + expect(output.data?.result?.content).to.equal(2); + }, + (): void => { + expect( + Object.entries(output.data?.result?.content[0])[0][1] + ).to.equal(2); + }, + (): void => { + expect( + Object.entries(output.data?.result?.content[0])[0][1] + ).to.equal('2'); + }, + ])(null); + }, + }, + { + testCase: 'Complex aggregation code generation', + type: 'generic', + userInput: + 'what percentage of recipes have "salt" in their ingredients? "ingredients" is a field ' + + 'with an array of strings of the ingredients. Only consider recipes ' + + 'that have the "difficulty Medium or Easy. Return is as a string named "saltPercentage" like ' + + '"75%", rounded to the nearest whole number. db CookBook, collection recipes', + assertResult: async ({ + responseContent, + connectionString, + }: AssertProps): Promise => { + const output = await runCodeInMessage(responseContent, connectionString); + + expect(output.data?.result?.content[0].saltPercentage).to.equal('67%'); + }, + }, +]; + +const testCases: TestCase[] = [ + ...namespaceTestCases, + ...queryTestCases, + ...intentTestCases, + ...genericTestCases, +]; + +const projectRoot = path.join(__dirname, '..', '..', '..'); + +const execFile = util.promisify(callbackExecFile); + +const TEST_RESULTS_DB = 'test_generative_ai_accuracy_evergreen'; +const TEST_RESULTS_COL = 'evergreen_runs'; + +const DEFAULT_ATTEMPTS_PER_TEST = 2; +const ATTEMPTS_PER_TEST = process.env.AI_TESTS_ATTEMPTS_PER_TEST + ? +process.env.AI_TESTS_ATTEMPTS_PER_TEST + : DEFAULT_ATTEMPTS_PER_TEST; + +/** + * Insert the generative ai results to a db + * so we can track how they perform overtime. + */ +async function pushResultsToDB({ + results, + anyFailedAccuracyThreshold, + runTimeMS, + httpErrors, +}: { + results: TestResult[]; + anyFailedAccuracyThreshold: boolean; + runTimeMS: number; + httpErrors: number; +}): Promise { + const client = new MongoClient( + process.env.AI_ACCURACY_RESULTS_MONGODB_CONNECTION_STRING || '' + ); + + try { + const database = client.db(TEST_RESULTS_DB); + const collection = database.collection(TEST_RESULTS_COL); + + const gitCommitHash = await execFile('git', ['rev-parse', 'HEAD'], { + cwd: projectRoot, + }); + + const doc = { + gitHash: gitCommitHash.stdout.trim(), + completedAt: new Date(), + attemptsPerTest: ATTEMPTS_PER_TEST, + anyFailedAccuracyThreshold, + httpErrors, + totalRunTimeMS: runTimeMS, // Total elapsed time including timeouts to avoid rate limit. + results: results.map((result) => { + const { 'Avg Execution Time (ms)': runTimeMS, Pass, ...rest } = result; + return { + runTimeMS, + Pass: Pass === '✓', + ...rest, + }; + }), + }; + + await collection.insertOne(doc); + } finally { + await client.close(); + } +} + +const buildMessages = async ({ + testCase, + fixtures, +}: { + testCase: TestCase; + fixtures: Fixtures; +}): Promise => { + switch (testCase.type) { + case 'intent': + return Prompts.intent.buildMessages({ + request: { prompt: testCase.userInput }, + context: { history: [] }, + connectionNames: [], + }); + + case 'generic': + return await Prompts.generic.buildMessages({ + request: { prompt: testCase.userInput }, + context: { history: [] }, + connectionNames: [], + }); + + case 'query': + return await Prompts.query.buildMessages({ + request: { prompt: testCase.userInput }, + context: { history: [] }, + databaseName: testCase.databaseName ?? 'test', + collectionName: testCase.collectionName ?? 'test', + connectionNames: [], + ...(fixtures[testCase.databaseName as string]?.[ + testCase.collectionName as string + ]?.schema + ? { + schema: + fixtures[testCase.databaseName as string]?.[ + testCase.collectionName as string + ]?.schema, + } + : {}), + ...(testCase.includeSampleDocuments + ? { + sampleDocuments: fixtures[testCase.databaseName as string][ + testCase.collectionName as string + ].documents.slice(0, 3), + } + : {}), + }); + + case 'namespace': + return Prompts.namespace.buildMessages({ + request: { prompt: testCase.userInput }, + context: { history: [] }, + connectionNames: [], + }); + + default: + throw new Error(`Unknown test case type: ${testCase.type}`); + } +}; + +async function runTest({ + testCase, + aiBackend, + fixtures, +}: { + testCase: TestCase; + aiBackend: AIBackend; + fixtures: Fixtures; +}): Promise { + const { messages } = await buildMessages({ + testCase, + fixtures, + }); + if (DEBUG_PROMPTS) { + console.log('Messages to send to chat completion:'); + console.log(messages); + } + const chatCompletion = await aiBackend.runAIChatCompletionGeneration({ + messages: messages.map((message) => ({ + ...message, + role: + message.role === vscode.LanguageModelChatMessageRole.User + ? 'user' + : 'assistant', + })), + }); + + return chatCompletion; +} + +describe('AI Accuracy Tests', function () { + let cluster: MongoCluster; + let mongoClient: MongoClient; + let fixtures: Fixtures = {}; + let anyFailedAccuracyThreshold = false; + let startTime; + let aiBackend; + let connectionString: string; + + const results: TestResult[] = []; + const testOutputs: TestOutputs = {}; + + this.timeout(60_000 /* 1 min */); + + before(async function () { + console.log('Starting setup for AI accuracy tests...'); + + const startupStartTime = Date.now(); + + cluster = await MongoCluster.start({ + tmpDir: os.tmpdir(), + topology: 'standalone', + }); + console.log('Started a test cluster:', cluster.connectionString); + connectionString = cluster.connectionString; + + mongoClient = new MongoClient(cluster.connectionString); + + fixtures = await loadFixturesToDB({ + mongoClient, + }); + + aiBackend = new AIBackend('openai'); + + console.log(`Test setup complete in ${Date.now() - startupStartTime}ms.`); + console.log('Starting AI accuracy tests...'); + startTime = Date.now(); + }); + + after(async function () { + console.log('Finished AI accuracy tests.'); + console.log('Results:', results); + + console.table(results, [ + 'Type', + 'Test', + 'Namespace', + 'Accuracy', + 'Avg Execution Time (ms)', + 'Avg Prompt Tokens', + 'Avg Completion Tokens', + 'Pass', + ]); + + if (process.env.AI_ACCURACY_RESULTS_MONGODB_CONNECTION_STRING) { + await pushResultsToDB({ + results, + anyFailedAccuracyThreshold, + httpErrors: 0, // TODO + runTimeMS: Date.now() - startTime, + }); + } + + await mongoClient?.close(); + await cluster?.close(); + + const htmlPageLocation = await createTestResultsHTMLPage({ + testResults: results, + testOutputs, + }); + console.log('View prompts and responses here:'); + console.log(htmlPageLocation); + }); + + for (const testCase of testCases) { + const testFunction = testCase.only ? it.only : it; + + testFunction( + `should pass for input: "${testCase.userInput}" if average accuracy is above threshold`, + // eslint-disable-next-line no-loop-func, complexity + async function () { + console.log(`Starting test run of ${testCase.testCase}.`); + + const testRunDidSucceed: boolean[] = []; + // Successful and unsuccessful runs are both tracked as long as the model + // returns a response. + const runStats: { + promptTokens: number; + completionTokens: number; + executionTimeMS: number; + }[] = []; + const accuracyThreshold = testCase.accuracyThresholdOverride ?? 0.8; + testOutputs[testCase.testCase] = { + prompt: testCase.userInput, + testType: testCase.type, + outputs: [], + }; + + for (let i = 0; i < numberOfRunsPerTest; i++) { + let success = false; + + if (testCase.reloadFixtureOnEachRun) { + await reloadFixture({ + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + db: testCase.databaseName!, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + coll: testCase.collectionName!, + mongoClient, + fixtures, + }); + } + + const startTime = Date.now(); + let responseContent: ChatCompletion | undefined; + let executionTimeMS = 0; + try { + responseContent = await runTest({ + testCase, + aiBackend, + fixtures, + }); + executionTimeMS = Date.now() - startTime; + testOutputs[testCase.testCase].outputs.push( + responseContent.content + ); + await testCase.assertResult({ + responseContent: responseContent.content, + connectionString, + fixtures, + mongoClient, + }); + + success = true; + + console.log( + `Test run of ${testCase.testCase}. Run ${i} of ${numberOfRunsPerTest} succeeded` + ); + } catch (err) { + console.log( + `Test run of ${testCase.testCase}. Run ${i} of ${numberOfRunsPerTest} failed with error:`, + err + ); + } + + if ( + responseContent && + responseContent?.usageStats?.completionTokens > 0 && + executionTimeMS !== 0 + ) { + runStats.push({ + completionTokens: responseContent.usageStats.completionTokens, + promptTokens: responseContent.usageStats.promptTokens, + executionTimeMS, + }); + } + + testRunDidSucceed.push(success); + } + + const averageAccuracy = + testRunDidSucceed.reduce((a, b) => a + (b ? 1 : 0), 0) / + testRunDidSucceed.length; + const didFail = averageAccuracy < accuracyThreshold; + + anyFailedAccuracyThreshold = anyFailedAccuracyThreshold || didFail; + + results.push({ + Test: testCase.testCase, + Type: testCase.type, + 'User Input': testCase.userInput.slice(0, 100), + Namespace: testCase.collectionName + ? `${testCase.databaseName}.${testCase.collectionName}` + : '', + Accuracy: averageAccuracy, + Pass: didFail ? '✗' : '✓', + 'Avg Execution Time (ms)': + runStats.length > 0 + ? runStats.reduce((a, b) => a + b.executionTimeMS, 0) / + runStats.length + : 0, + 'Avg Prompt Tokens': + runStats.length > 0 + ? runStats.reduce((a, b) => a + b.promptTokens, 0) / + runStats.length + : 0, + 'Avg Completion Tokens': + runStats.length > 0 + ? runStats.reduce((a, b) => a + b.completionTokens, 0) / + runStats.length + : 0, + }); + + expect(averageAccuracy).to.be.at.least( + accuracyThreshold, + `Average accuracy (${averageAccuracy}) for input "${testCase.userInput}" is below the threshold (${accuracyThreshold})` + ); + } + ); + } +}); diff --git a/src/test/ai-accuracy-tests/ai-backend.ts b/src/test/ai-accuracy-tests/ai-backend.ts new file mode 100644 index 000000000..02534f580 --- /dev/null +++ b/src/test/ai-accuracy-tests/ai-backend.ts @@ -0,0 +1,103 @@ +import OpenAI from 'openai'; +import type { ChatCompletionCreateParamsBase } from 'openai/resources/chat/completions'; + +import { CHAT_PARTICIPANT_MODEL } from '../../participant/constants'; + +let openai: OpenAI; +function getOpenAIClient(): OpenAI { + if (!openai) { + openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + }); + } + + return openai; +} + +export type AIService = 'openai'; + +type ChatMessage = { + role: 'user' | 'assistant'; + content: string; +}; +type ChatMessages = ChatMessage[]; + +export type ChatCompletion = { + content: string; + usageStats: { + promptTokens: number; + completionTokens: number; + }; +}; + +async function createOpenAIChatCompletion({ + messages, + model = CHAT_PARTICIPANT_MODEL, +}: { + messages: ChatMessages; + model?: ChatCompletionCreateParamsBase['model']; +}): Promise { + const openai = getOpenAIClient(); + + // TODO: Currently we aren't supplying a system message, that may + // create a discrepancy in responses. We should investigate passing a system + // message, even if it's minimal. + const completion: OpenAI.Chat.Completions.ChatCompletion = + await openai.chat.completions.create({ + messages, + model, + }); + + return { + content: completion.choices[0].message.content || '', + usageStats: { + promptTokens: completion.usage?.prompt_tokens ?? NaN, + completionTokens: completion.usage?.completion_tokens ?? NaN, + }, + }; +} + +export type UsageStats = { promptTokens: number; completionTokens: number }; + +export type GenerationResponse = { + content: string; + query?: { + filter?: string; + project?: string; + sort?: string; + limit?: string; + skip?: string; + }; + aggregation?: string; + usageStats?: UsageStats; +}; + +export function createAIChatCompletion({ + messages, +}: { + messages: ChatMessages; + backend?: AIService; +}): Promise { + // Defaults to open ai for now + return createOpenAIChatCompletion({ messages }); +} + +export class AIBackend { + aiService: AIService; + + constructor(aiService: AIService) { + this.aiService = aiService; + } + + async runAIChatCompletionGeneration({ + messages, + }: { + messages: ChatMessages; + }): Promise { + const completion = await createAIChatCompletion({ + messages, + backend: this.aiService, + }); + return completion; + } +} diff --git a/src/test/ai-accuracy-tests/assertions.ts b/src/test/ai-accuracy-tests/assertions.ts new file mode 100644 index 000000000..304cc68b8 --- /dev/null +++ b/src/test/ai-accuracy-tests/assertions.ts @@ -0,0 +1,102 @@ +import assert from 'assert'; +import util from 'util'; +import type { Document } from 'mongodb'; + +import type { Fixtures } from './fixtures/fixture-loader'; +import { execute } from '../../language/worker'; +import type { ShellEvaluateResult } from '../../types/playgroundType'; +import { asyncIterableFromArray } from '../suite/participant/asyncIterableFromArray'; +import { codeBlockIdentifier } from '../../participant/constants'; +import { processStreamWithIdentifiers } from '../../participant/streamParsing'; + +export const runCodeInMessage = async ( + message: string, + connectionString: string +): Promise<{ + printOutput: string[]; + data: ShellEvaluateResult; + error: any; +}> => { + // We only run the last code block passed. + let codeToEvaluate = ''; + await processStreamWithIdentifiers({ + processStreamFragment: () => { + /* no-op */ + }, + onStreamIdentifier: (codeBlockContent: string): void => { + codeToEvaluate = codeBlockContent; + }, + inputIterable: asyncIterableFromArray([message]), + identifier: codeBlockIdentifier, + }); + + if (codeToEvaluate.trim().length === 0) { + throw new Error(`no code found in message: ${message}`); + } + + const printOutput: string[] = []; + + const { data, error } = await execute({ + codeToEvaluate, + connectionString, + connectionOptions: { + productName: 'VSCode Copilot AI accuracy tests', + productDocsLink: 'N/A', + }, + onPrint: (values) => { + printOutput.push( + ...values.map((v) => + typeof v.printable === 'string' + ? v.printable + : util.inspect(v.printable) + ) + ); + }, + }); + + if (error) { + throw new Error( + `An error occurred when attempting to run the code in the message: \n${message}\n___Error:\n${error}` + ); + } + + return { + printOutput, + data, + error, + }; +}; + +export const isDeepStrictEqualTo = + (expected: unknown) => + (actual: unknown): void => + assert.deepStrictEqual(actual, expected); + +export const isDeepStrictEqualToFixtures = + ( + db: string, + coll: string, + fixtures: Fixtures, + comparator: (document: Document) => boolean + ) => + (actual: unknown): void => { + const expected = fixtures[db][coll].documents.filter(comparator); + assert.deepStrictEqual(actual, expected); + }; + +export const anyOf = + (assertions: ((result: unknown) => void)[]) => + (actual: unknown): void => { + const errors: Error[] = []; + for (const assertion of assertions) { + try { + assertion(actual); + } catch (e) { + errors.push(e as Error); + } + } + + if (errors.length === assertions.length) { + throw errors[errors.length - 1]; + } + }; diff --git a/src/test/ai-accuracy-tests/create-test-results-html-page.ts b/src/test/ai-accuracy-tests/create-test-results-html-page.ts new file mode 100644 index 000000000..c3774a59e --- /dev/null +++ b/src/test/ai-accuracy-tests/create-test-results-html-page.ts @@ -0,0 +1,112 @@ +import { promises as fs } from 'fs'; +import path from 'path'; + +export type TestResult = { + Test: string; + Type: string; + 'User Input': string; + Namespace: string; + Accuracy: number; + Pass: '✗' | '✓'; + 'Avg Execution Time (ms)': number; + 'Avg Prompt Tokens': number; + 'Avg Completion Tokens': number; +}; + +type TestOutput = { + prompt: string; + testType: string; + outputs: string[]; +}; + +export type TestOutputs = { + [testName: string]: TestOutput; +}; + +const createTestLinkId = (testName: string): string => + encodeURIComponent(testName.replace(/ /g, '-')); + +function getTestResultsTable(testResults: TestResult[]): string { + const headers = Object.keys(testResults[0]) + .map((key) => `${key}`) + .join(''); + + const resultRows = testResults + .map((result) => { + const row = Object.entries(result) + .map( + ([field, value]) => + `${ + field === 'Test' + ? `${value}` + : value + }` + ) + .join(''); + return `${row}`; + }) + .join(''); + + return ` + + + ${headers} + + + ${resultRows} + +
+`; +} + +function getTestOutputTables(testOutputs: TestOutputs): string { + const outputTables = Object.entries(testOutputs) + .map(([testName, output]) => { + const outputRows = output.outputs + .map((out) => `${out}`) + .join(''); + return ` + +

Prompt: ${output.prompt}

+ + + + + + ${outputRows} + +
Outputs
+ `; + }) + .join(''); + + return outputTables; +} + +export async function createTestResultsHTMLPage({ + testResults, + testOutputs, +}: { + testResults: TestResult[]; + testOutputs: TestOutputs; +}): Promise { + const htmlOutput = ` + + Test Results + + + +

Test Results

+ ${getTestResultsTable(testResults)} +

Test Outputs

+ ${getTestOutputTables(testOutputs)} + +`; + + const htmlPageLocation = path.join(__dirname, 'test-results.html'); + await fs.writeFile(htmlPageLocation, htmlOutput); + + return htmlPageLocation; +} diff --git a/src/test/ai-accuracy-tests/fixtures/antiques.ts b/src/test/ai-accuracy-tests/fixtures/antiques.ts new file mode 100644 index 000000000..81c5ef1d8 --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/antiques.ts @@ -0,0 +1,74 @@ +import type { Fixture } from './fixture-type'; + +const antiques: Fixture = { + db: 'Antiques', + coll: 'items', + documents: [ + { + itemName: 'Vintage Beatles Vinyl', + owner: { + name: 'John Doe', + location: 'London', + }, + acquisition: { + date: '1998-03-13', + price: 1200, + }, + condition: 'Mint', + history: [ + { event: 'Auction Win', date: '1998-03-13' }, + { event: 'Restoration', date: '2005-07-22' }, + ], + }, + { + itemName: 'Ancient Roman Coin', + owner: { + name: 'Jane Doe', + location: 'Rome', + }, + acquisition: { + date: '2002-11-27', + price: 5000, + }, + condition: 'Good', + history: [ + { event: 'Found in a dig', date: '2002-11-27' }, + { event: 'Museum Display', date: '2010-02-15' }, + ], + }, + { + itemName: 'Victorian Pocket Watch', + owner: { + name: 'Arnold Arnoldson', + location: 'London', + }, + acquisition: { + date: '2010-06-30', + price: 800, + }, + condition: 'Fair', + history: [ + { event: 'Inherited', date: '2010-06-30' }, + { event: 'Repair', date: '2015-09-12' }, + ], + }, + { + itemName: 'An Ancient Pineapple (super rare)', + owner: { + name: 'Monkey', + location: 'New York', + }, + acquisition: { + date: '2018-02-05', + price: 2300, + }, + condition: 'Mint', + history: [ + { event: 'Estate Sale', date: '2018-02-05' }, + { event: 'Appraisal', date: '2020-04-18' }, + ], + }, + ], +}; + +export default antiques; diff --git a/src/test/ai-accuracy-tests/fixtures/fixture-loader.ts b/src/test/ai-accuracy-tests/fixtures/fixture-loader.ts new file mode 100644 index 000000000..a97978a34 --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/fixture-loader.ts @@ -0,0 +1,71 @@ +import type { Document, MongoClient } from 'mongodb'; +import { getSimplifiedSchema } from 'mongodb-schema'; + +import type { Fixture } from './fixture-type'; +import antiques from './antiques'; +import pets from './pets'; +import pineapples from './pineapples'; +import recipes from './recipes'; +import getUFOSightingsFixture from './ufo'; +import { SchemaFormatter } from '../../../participant/schema'; + +export type Fixtures = { + [dbName: string]: { + [colName: string]: { + documents: Document[]; + schema: string; // Result of formatted simplified schema. + }; + }; +}; + +type LoadableFixture = (() => Fixture) | Fixture; +const fixturesToLoad: LoadableFixture[] = [ + antiques, + pets, + pineapples, + recipes, + getUFOSightingsFixture, +]; + +export async function loadFixturesToDB({ + mongoClient, +}: { + mongoClient: MongoClient; +}): Promise { + const fixtures: Fixtures = {}; + + // Load dynamic fixtures. + for (const fixtureToLoad of fixturesToLoad) { + const { db, coll, documents } = + typeof fixtureToLoad === 'function' ? fixtureToLoad() : fixtureToLoad; + + const unformattedSchema = await getSimplifiedSchema(documents); + const schema = new SchemaFormatter().format(unformattedSchema); + + fixtures[db] = { + [coll]: { + documents, + schema, + }, + }; + await mongoClient.db(db).collection(coll).insertMany(documents); + } + + return fixtures; +} + +export async function reloadFixture({ + db, + coll, + mongoClient, + fixtures, +}: { + db: string; + coll: string; + mongoClient: MongoClient; + fixtures: Fixtures; +}): Promise { + await mongoClient.db(db).collection(coll).drop(); + const { documents } = fixtures[db][coll]; + await mongoClient.db(db).collection(coll).insertMany(documents); +} diff --git a/src/test/ai-accuracy-tests/fixtures/fixture-type.ts b/src/test/ai-accuracy-tests/fixtures/fixture-type.ts new file mode 100644 index 000000000..dfa3d38e3 --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/fixture-type.ts @@ -0,0 +1,7 @@ +import type { Document } from 'mongodb'; + +export type Fixture = { + db: string; + coll: string; + documents: Document[]; +}; diff --git a/src/test/ai-accuracy-tests/fixtures/pets.ts b/src/test/ai-accuracy-tests/fixtures/pets.ts new file mode 100644 index 000000000..50c66748d --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/pets.ts @@ -0,0 +1,38 @@ +import type { Fixture } from './fixture-type'; + +const petCompetition: Fixture = { + db: 'pets', + coll: 'competition-results', + documents: [ + { + name: 'Fluffy', + species: 'dog', + category: 'best costume', + score: 9.1, + year: 2021, + }, + { + name: 'Scruffy', + species: 'dog', + category: 'best costume', + score: 9.5, + year: 2021, + }, + { + name: 'Whiskers', + species: 'cat', + category: 'most agile', + score: 8.7, + year: 2022, + }, + { + name: 'Bubbles', + species: 'fish', + category: 'prettiest scales', + score: 7.5, + year: 2021, + }, + ], +}; + +export default petCompetition; diff --git a/src/test/ai-accuracy-tests/fixtures/pineapples.ts b/src/test/ai-accuracy-tests/fixtures/pineapples.ts new file mode 100644 index 000000000..a1ae1501f --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/pineapples.ts @@ -0,0 +1,34 @@ +const pineapples = { + db: 'FarmData', + coll: 'Pineapples', + documents: [ + { + weightKg: 2.4, + heightCm: 25, + plantedDate: '2022-03-15', + harvestedDate: '2022-09-20', + soilPH: 5.5, + farmerNotes: 'Grew faster than usual due to experimental fertilizer', + sweetnessScale: 8, + color: 'Golden', + waterings: 35, + sunlightHours: 400, + pestIncidents: 2, + }, + { + weightKg: 1.8, + heightCm: 22, + plantedDate: '2021-11-10', + harvestedDate: '2022-06-05', + soilPH: 6.0, + farmerNotes: 'Had issues with pests but used organic methods to control', + sweetnessScale: 7, + color: 'Yellow', + waterings: 28, + sunlightHours: 380, + pestIncidents: 3, + }, + ], +}; + +export default pineapples; diff --git a/src/test/ai-accuracy-tests/fixtures/recipes.ts b/src/test/ai-accuracy-tests/fixtures/recipes.ts new file mode 100644 index 000000000..f9e8d7346 --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/recipes.ts @@ -0,0 +1,55 @@ +import type { Fixture } from './fixture-type'; + +const recipes: Fixture = { + db: 'CookBook', + coll: 'recipes', + documents: [ + { + title: 'Spaghetti Bolognese', + ingredients: [ + 'spaghetti', + 'ground beef', + 'tomato sauce', + 'onions', + 'garlic', + 'salt', + ], + preparationTime: 60, + difficulty: 'Medium', + }, + { + title: 'Avocado Toast', + ingredients: ['avocado', 'bread', 'salt', 'pepper'], + optionalIngredients: ['lime'], + preparationTime: 10, + difficulty: 'Easy', + }, + { + title: 'Pineapple', + ingredients: ['pineapple'], + preparationTime: 5, + difficulty: 'Very Hard', + }, + { + title: 'Pizza', + ingredients: ['dough', 'tomato sauce', 'mozzarella cheese', 'basil'], + optionalIngredients: ['pineapple'], + preparationTime: 40, + difficulty: 'Medium', + }, + { + title: 'Beef Wellington', + ingredients: [ + 'beef tenderloin', + 'mushroom duxelles', + 'puff pastry', + 'egg wash', + 'salt', + ], + preparationTime: 120, + difficulty: 'Hard', + }, + ], +}; + +export default recipes; diff --git a/src/test/ai-accuracy-tests/fixtures/ufo.ts b/src/test/ai-accuracy-tests/fixtures/ufo.ts new file mode 100644 index 000000000..5f7809514 --- /dev/null +++ b/src/test/ai-accuracy-tests/fixtures/ufo.ts @@ -0,0 +1,39 @@ +import type { Fixture } from './fixture-type'; + +// Dynamic date fixture. +export default function getUFOSightingsFixture(): Fixture { + return { + db: 'UFO', + coll: 'sightings', + documents: [ + { + description: 'Flying Saucer in the sky, numerous reports.', + where: 'Oklahoma', + // Last year. + year: `${new Date().getFullYear() - 1}`, + }, + { + description: 'Alien spaceship.', + where: 'Tennessee', + year: '2005', + }, + { + description: + 'Portal in the sky created by moving object, possibly just northern lights.', + where: 'Alaska', + year: '2020', + }, + { + description: 'Floating pineapple, likely northern lights.', + where: 'Alaska', + year: '2021', + }, + { + description: + 'Someone flying on a broomstick, sighters reported "It looks like Harry Potter".', + where: 'New York', + year: '2022', + }, + ], + }; +} diff --git a/src/test/ai-accuracy-tests/test-results-page-styles.css b/src/test/ai-accuracy-tests/test-results-page-styles.css new file mode 100644 index 000000000..3dd8ccc78 --- /dev/null +++ b/src/test/ai-accuracy-tests/test-results-page-styles.css @@ -0,0 +1,27 @@ +table { + width: 100%; + border-collapse: collapse; + margin-bottom: 24px; +} +th, +td { + border: 1px solid #ddd; + padding: 8px; + white-space: pre-wrap; +} +th { + background-color: #f2f2f2; + text-align: left; +} +tr:nth-child(even) { + background-color: #f9f9f9; +} +tr:hover td { + border-left-color: #777; + border-right-color: #777; +} +h1, +h2 { + font-family: Arial, sans-serif; + white-space: pre-wrap; +} diff --git a/src/test/ai-accuracy-tests/test-setup.ts b/src/test/ai-accuracy-tests/test-setup.ts new file mode 100644 index 000000000..bd0a4d217 --- /dev/null +++ b/src/test/ai-accuracy-tests/test-setup.ts @@ -0,0 +1,40 @@ +import rewiremock from 'rewiremock'; + +const AssistantRole = 2; +const UserRole = 1; +const vscodeMock = { + LanguageModelChatMessageRole: { + Assistant: AssistantRole, + User: UserRole, + }, + LanguageModelChatMessage: { + Assistant: (content, name?: string): unknown => ({ + name, + content, + role: AssistantRole, + }), + User: (content: string, name?: string): unknown => ({ + content, + name, + role: UserRole, + }), + }, + window: { + createOutputChannel: (): void => {}, + }, + lm: { + selectChatModels: (): unknown => [ + { + countTokens: (input: string): number => { + return input.length; + }, + maxInputTokens: 10_000, + }, + ], + }, +}; + +// Mock the 'vscode' module since we don't run the full vscode +// integration test setup for the ai-accuracy-tests as it's a bit slow. +rewiremock('vscode').with(vscodeMock); +rewiremock.enable(); diff --git a/src/test/suite/editors/playgroundController.test.ts b/src/test/suite/editors/playgroundController.test.ts index 2a0e44e12..125ba4a7e 100644 --- a/src/test/suite/editors/playgroundController.test.ts +++ b/src/test/suite/editors/playgroundController.test.ts @@ -79,7 +79,6 @@ suite('Playground Controller Test Suite', function () { testExportToLanguageCodeLensProvider = new ExportToLanguageCodeLensProvider(); testCodeActionProvider = new PlaygroundSelectedCodeActionProvider(); - languageServerControllerStub = new LanguageServerControllerStub( extensionContextStub, testStorageController @@ -325,8 +324,9 @@ suite('Playground Controller Test Suite', function () { sandbox.fake.rejects(false) ); - const result = - await testPlaygroundController._evaluateWithCancelModal(); + const result = await testPlaygroundController._evaluateWithCancelModal({ + codeToEvaluate: '', + }); expect(result).to.deep.equal({ result: undefined }); }); @@ -399,7 +399,7 @@ suite('Playground Controller Test Suite', function () { ); sandbox.replace( testPlaygroundController, - '_openPlaygroundResult', + '_openInResultPane', sandbox.stub() ); }); diff --git a/src/test/suite/editors/playgroundSelectedCodeActionProvider.test.ts b/src/test/suite/editors/playgroundSelectedCodeActionProvider.test.ts index 735ce1a0b..774f912ca 100644 --- a/src/test/suite/editors/playgroundSelectedCodeActionProvider.test.ts +++ b/src/test/suite/editors/playgroundSelectedCodeActionProvider.test.ts @@ -72,7 +72,7 @@ suite('Playground Selected CodeAction Provider Test Suite', function () { const fakeOpenPlaygroundResult = sandbox.fake(); sandbox.replace( mdbTestExtension.testExtensionController._playgroundController, - '_openPlaygroundResult', + '_openInResultPane', fakeOpenPlaygroundResult ); diff --git a/src/test/suite/mdbExtensionController.test.ts b/src/test/suite/mdbExtensionController.test.ts index e887485b9..5e11b1cc3 100644 --- a/src/test/suite/mdbExtensionController.test.ts +++ b/src/test/suite/mdbExtensionController.test.ts @@ -33,7 +33,7 @@ const testDatabaseURI = 'mongodb://localhost:27088'; function getTestConnectionTreeItem( options?: Partial[0]> -) { +): ConnectionTreeItem { return new ConnectionTreeItem({ connectionId: 'tasty_sandwhich', collapsibleState: vscode.TreeItemCollapsibleState.None, @@ -48,7 +48,7 @@ function getTestConnectionTreeItem( function getTestCollectionTreeItem( options?: Partial[0]> -) { +): CollectionTreeItem { return new CollectionTreeItem({ collection: { name: 'testColName', @@ -65,7 +65,7 @@ function getTestCollectionTreeItem( function getTestDatabaseTreeItem( options?: Partial[0]> -) { +): DatabaseTreeItem { return new DatabaseTreeItem({ databaseName: 'zebra', dataService: {} as DataService, @@ -78,7 +78,7 @@ function getTestDatabaseTreeItem( function getTestStreamProcessorTreeItem( options?: Partial[0]> -) { +): StreamProcessorTreeItem { return new StreamProcessorTreeItem({ streamProcessorName: 'zebra', streamProcessorState: 'CREATED', @@ -88,7 +88,7 @@ function getTestStreamProcessorTreeItem( }); } -function getTestFieldTreeItem() { +function getTestFieldTreeItem(): FieldTreeItem { return new FieldTreeItem({ field: { name: 'dolphins are sentient', @@ -101,7 +101,7 @@ function getTestFieldTreeItem() { }); } -function getTestSchemaTreeItem() { +function getTestSchemaTreeItem(): SchemaTreeItem { return new SchemaTreeItem({ databaseName: 'zebraWearwolf', collectionName: 'giraffeVampire', @@ -116,7 +116,7 @@ function getTestSchemaTreeItem() { function getTestDocumentTreeItem( options?: Partial[0]> -) { +): DocumentTreeItem { return new DocumentTreeItem({ document: {}, namespace: 'waffle.house', @@ -129,10 +129,14 @@ function getTestDocumentTreeItem( suite('MDBExtensionController Test Suite', function () { this.timeout(10000); + const sandbox = sinon.createSandbox(); + + afterEach(() => { + sandbox.restore(); + }); suite('when not connected', () => { let showErrorMessageStub: SinonSpy; - const sandbox = sinon.createSandbox(); beforeEach(() => { sandbox.stub(vscode.window, 'showInformationMessage'); @@ -145,10 +149,6 @@ suite('MDBExtensionController Test Suite', function () { ); }); - afterEach(() => { - sandbox.restore(); - }); - test('mdb.addDatabase command fails when not connected to the connection', async () => { const testTreeItem = getTestConnectionTreeItem(); const addDatabaseSucceeded = await vscode.commands.executeCommand( @@ -177,8 +177,6 @@ suite('MDBExtensionController Test Suite', function () { let fakeCreatePlaygroundFileWithContent: SinonSpy; let openExternalStub: SinonStub; - const sandbox = sinon.createSandbox(); - beforeEach(() => { showInformationMessageStub = sandbox.stub( vscode.window, @@ -206,10 +204,6 @@ suite('MDBExtensionController Test Suite', function () { ); }); - afterEach(() => { - sandbox.restore(); - }); - test('mdb.viewCollectionDocuments command should call onViewCollectionDocuments on the editor controller with the collection namespace', async () => { const textCollectionTree = getTestCollectionTreeItem(); await vscode.commands.executeCommand( @@ -1853,4 +1847,56 @@ suite('MDBExtensionController Test Suite', function () { }); }); }); + + test('mdb.participantViewRawSchemaOutput command opens a json document with the output', async () => { + const openTextDocumentStub = sandbox.stub( + vscode.workspace, + 'openTextDocument' + ); + const showTextDocumentStub = sandbox.stub( + vscode.window, + 'showTextDocument' + ); + + const schemaContent = `{ + "count": 1, + "fields": [ + { + "name": "_id", + "path": [ + "_id" + ], + "count": 1, + "type": "ObjectId", + "probability": 1, + "hasDuplicates": false, + "types": [ + { + "name": "ObjectId", + "path": [ + "_id" + ], + "count": 1, + "probability": 1, + "bsonType": "ObjectId" + } + ] + } + ] +}`; + await vscode.commands.executeCommand('mdb.participantViewRawSchemaOutput', { + schema: schemaContent, + }); + + assert(openTextDocumentStub.calledOnce); + assert.deepStrictEqual(openTextDocumentStub.firstCall.args[0], { + language: 'json', + content: schemaContent, + }); + + assert(showTextDocumentStub.calledOnce); + assert.deepStrictEqual(showTextDocumentStub.firstCall.args[1], { + preview: true, + }); + }); }); diff --git a/src/test/suite/participant/asyncIterableFromArray.ts b/src/test/suite/participant/asyncIterableFromArray.ts new file mode 100644 index 000000000..e3b7c8bde --- /dev/null +++ b/src/test/suite/participant/asyncIterableFromArray.ts @@ -0,0 +1,24 @@ +// Exported here so that the accuracy tests can use it without +// needing to define all of the testing types the main tests have. +export function asyncIterableFromArray(array: T[]): AsyncIterable { + return { + [Symbol.asyncIterator](): { + next(): Promise>; + } { + let index = 0; + return { + next(): Promise<{ + value: any; + done: boolean; + }> { + if (index < array.length) { + const value = array[index++]; + return Promise.resolve({ value, done: false }); + } + + return Promise.resolve({ value: undefined, done: true }); + }, + }; + }, + }; +} diff --git a/src/test/suite/participant/docsChatbotAIService.test.ts b/src/test/suite/participant/docsChatbotAIService.test.ts new file mode 100644 index 000000000..e0f97e7ff --- /dev/null +++ b/src/test/suite/participant/docsChatbotAIService.test.ts @@ -0,0 +1,141 @@ +import { beforeEach, afterEach } from 'mocha'; +import { expect } from 'chai'; +import sinon from 'sinon'; + +import { DocsChatbotAIService } from '../../../participant/docsChatbotAIService'; + +suite('DocsChatbotAIService Test Suite', function () { + const initialFetch = global.fetch; + let docsChatbotAIService: DocsChatbotAIService; + + beforeEach(() => { + docsChatbotAIService = new DocsChatbotAIService(); + }); + + afterEach(function () { + global.fetch = initialFetch; + sinon.restore(); + }); + + test('creates conversations', async () => { + const fetchStub = sinon.stub().resolves({ + status: 200, + ok: true, + json: () => + Promise.resolve({ + _id: '650b4b260f975ef031016c8a', + messages: [], + }), + }); + global.fetch = fetchStub; + const conversation = await docsChatbotAIService.createConversation({ + signal: new AbortController().signal, + }); + expect(conversation._id).to.be.eql('650b4b260f975ef031016c8a'); + }); + + test('throws on server errors', async () => { + const fetchStub = sinon.stub().resolves({ + status: 500, + ok: false, + statusText: 'Server error', + json: sinon.stub().rejects(new Error('invalid json')), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation({ + signal: new AbortController().signal, + }); + expect.fail('It must fail with the server error'); + } catch (error) { + expect((error as Error).message).to.include('Internal server error'); + } + }); + + test('throws when aborted', async () => { + try { + const abortController = new AbortController(); + abortController.abort(); + await docsChatbotAIService.createConversation({ + signal: abortController.signal, + }); + expect.fail('It must fail with the server error'); + } catch (error) { + expect((error as Error).message).to.include('This operation was aborted'); + } + }); + + test('throws on bad requests', async () => { + const fetchStub = sinon.stub().resolves({ + status: 400, + ok: false, + statusText: 'Client error', + json: sinon.stub().resolves({}), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation({ + signal: new AbortController().signal, + }); + expect.fail('It must fail with the bad request error'); + } catch (error) { + expect((error as Error).message).to.include('Bad request'); + } + }); + + test('throws on a rate limit', async () => { + const fetchStub = sinon.stub().resolves({ + status: 429, + ok: false, + statusText: 'Model error', + json: sinon.stub().resolves({}), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation({ + signal: new AbortController().signal, + }); + expect.fail('It must fail with the rate limited error'); + } catch (error) { + expect((error as Error).message).to.include('Rate limited'); + } + }); + + test('throws on timeout', async () => { + const fetchStub = sinon.stub().resolves({ + status: 504, + ok: false, + json: sinon.stub().resolves({}), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.addMessage({ + conversationId: '650b4b260f975ef031016c8a', + message: 'what is mongosh?', + signal: new AbortController().signal, + }); + expect.fail('It must fail with the timeout error'); + } catch (error) { + expect((error as Error).message).to.include('Timeout'); + } + }); + + test('rates docs chatbot response', async () => { + const fetchStub = sinon.stub().resolves({ + status: 204, + ok: true, + json: () => Promise.resolve(true), + }); + global.fetch = fetchStub; + const rating = await docsChatbotAIService.rateMessage({ + conversationId: '650b4b260f975ef031016c8a', + messageId: '1', + rating: true, + }); + expect(rating).to.be.eql(true); + }); +}); diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts new file mode 100644 index 000000000..12ba8ae46 --- /dev/null +++ b/src/test/suite/participant/participant.test.ts @@ -0,0 +1,1992 @@ +import * as vscode from 'vscode'; +import { beforeEach, afterEach } from 'mocha'; +import { expect } from 'chai'; +import type { SinonSpy } from 'sinon'; +import sinon from 'sinon'; +import type { DataService } from 'mongodb-data-service'; +import { ObjectId, Int32 } from 'bson'; + +import ParticipantController from '../../../participant/participant'; +import ConnectionController from '../../../connectionController'; +import { StorageController } from '../../../storage'; +import { StatusView } from '../../../views'; +import { ExtensionContextStub } from '../stubs'; +import type { + InternalPromptPurpose, + ParticipantPromptProperties, + ParticipantResponseProperties, +} from '../../../telemetry/telemetryService'; +import TelemetryService, { + TelemetryEventTypes, +} from '../../../telemetry/telemetryService'; +import { TEST_DATABASE_URI } from '../dbTestHelper'; +import type { ChatResult } from '../../../participant/constants'; +import { CHAT_PARTICIPANT_ID } from '../../../participant/constants'; +import { + SecretStorageLocation, + StorageLocation, +} from '../../../storage/storageController'; +import type { LoadedConnection } from '../../../storage/connectionStorage'; +import { ChatMetadataStore } from '../../../participant/chatMetadata'; +import { Prompts } from '../../../participant/prompts'; +import { createMarkdownLink } from '../../../participant/markdown'; +import EXTENSION_COMMANDS from '../../../commands'; + +// The Copilot's model in not available in tests, +// therefore we need to mock its methods and returning values. +const MAX_TOTAL_PROMPT_LENGTH_MOCK = 16000; + +const loadedConnection = { + id: 'id', + name: 'localhost', + storageLocation: StorageLocation.NONE, + secretStorageLocation: SecretStorageLocation.SecretStorage, + connectionOptions: { connectionString: 'mongodb://localhost' }, +}; + +const testChatId = 'test-chat-id'; + +const encodeStringify = (obj: Record): string => { + return encodeURIComponent(JSON.stringify(obj)); +}; + +suite('Participant Controller Test Suite', function () { + const extensionContextStub = new ExtensionContextStub(); + + // The test extension runner. + extensionContextStub.extensionPath = '../../'; + + let testConnectionController: ConnectionController; + let testStorageController: StorageController; + let testStatusView: StatusView; + let testTelemetryService: TelemetryService; + let testParticipantController: ParticipantController; + let chatContextStub: vscode.ChatContext; + let chatStreamStub: { + push: sinon.SinonSpy; + markdown: sinon.SinonSpy; + button: sinon.SinonSpy; + }; + let chatTokenStub; + let countTokensStub; + let sendRequestStub: sinon.SinonStub; + let telemetryTrackStub: SinonSpy; + + const invokeChatHandler = async ( + request: vscode.ChatRequest + ): Promise => + testParticipantController.chatHandler( + request, + chatContextStub, + chatStreamStub as unknown as vscode.ChatResponseStream, + chatTokenStub + ); + + const assertCommandTelemetry = ( + command: string, + chatRequest: vscode.ChatRequest, + { + expectSampleDocs = false, + callIndex = 0, + expectedInternalPurpose = undefined, + }: { + expectSampleDocs?: boolean; + callIndex: number; + expectedInternalPurpose?: InternalPromptPurpose; + } + ): void => { + expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); + + const call = telemetryTrackStub.getCalls()[callIndex]; + expect(call.args[0]).to.equal('Participant Prompt Submitted'); + + const properties = call.args[1] as ParticipantPromptProperties; + + expect(properties.command).to.equal(command); + expect(properties.has_sample_documents).to.equal(expectSampleDocs); + expect(properties.history_size).to.equal(chatContextStub.history.length); + + // Total message length includes participant as well as user prompt + expect(properties.total_message_length).to.be.greaterThan( + properties.user_input_length + ); + + // User prompt length should be at least equal to the supplied user prompt, but my occasionally + // be greater - e.g. when we enhance the context. + expect(properties.user_input_length).to.be.greaterThanOrEqual( + chatRequest.prompt.length + ); + expect(properties.internal_purpose).to.equal(expectedInternalPurpose); + }; + + const assertResponseTelemetry = ( + command: string, + { + callIndex = 0, + hasCTA = false, + hasRunnableContent = false, + foundNamespace = false, + }: { + callIndex: number; + hasCTA?: boolean; + hasRunnableContent?: boolean; + foundNamespace?: boolean; + } + ): void => { + expect(telemetryTrackStub.callCount).to.be.greaterThan(callIndex); + const call = telemetryTrackStub.getCalls()[callIndex]; + expect(call.args[0]).to.equal('Participant Response Generated'); + + const properties = call.args[1] as ParticipantResponseProperties; + + expect(properties.command).to.equal(command); + expect(properties.found_namespace).to.equal(foundNamespace); + expect(properties.has_cta).to.equal(hasCTA); + expect(properties.has_runnable_content).to.equal(hasRunnableContent); + expect(properties.output_length).to.be.greaterThan(0); + }; + + beforeEach(function () { + testStorageController = new StorageController(extensionContextStub); + testStatusView = new StatusView(extensionContextStub); + + telemetryTrackStub = sinon.stub(); + + testTelemetryService = new TelemetryService( + testStorageController, + extensionContextStub + ); + testConnectionController = new ConnectionController({ + statusView: testStatusView, + storageController: testStorageController, + telemetryService: testTelemetryService, + }); + sinon.replace(ChatMetadataStore, 'createNewChatId', () => testChatId); + testParticipantController = new ParticipantController({ + connectionController: testConnectionController, + storageController: testStorageController, + telemetryService: testTelemetryService, + }); + chatContextStub = { + history: [ + { + participant: CHAT_PARTICIPANT_ID, + prompt: 'hi', + response: [new vscode.ChatResponseMarkdownPart('hello')], + result: {}, + }, + ], + }; + chatStreamStub = { + push: sinon.fake(), + markdown: sinon.fake(), + button: sinon.fake(), + }; + chatTokenStub = { + onCancellationRequested: sinon.fake(), + }; + countTokensStub = sinon.stub(); + // The model returned by vscode.lm.selectChatModels is always undefined in tests. + sendRequestStub = sinon.stub().resolves({ + text: [ + '```javascript\n' + + "use('dbOne');\n" + + "db.getCollection('collOne').find({ name: 'example' });\n" + + '```', + ], + }); + sinon.replace( + vscode.lm, + 'selectChatModels', + sinon.fake.returns([ + { + id: 'modelId', + vendor: 'copilot', + family: 'gpt-4o', + version: 'gpt-4o-date', + name: 'GPT 4o (date)', + maxInputTokens: MAX_TOTAL_PROMPT_LENGTH_MOCK, + countTokens: countTokensStub, + sendRequest: sendRequestStub, + }, + ]) + ); + + sinon.replace(testTelemetryService, 'track', telemetryTrackStub); + }); + + afterEach(function () { + sinon.restore(); + }); + + test('parses a returned by ai text for database and collection name', function () { + const text = 'DATABASE_NAME: my \nCOLLECTION_NAME: cats'; + const { databaseName, collectionName } = + Prompts.namespace.extractDatabaseAndCollectionNameFromResponse(text); + expect(databaseName).to.be.equal('my'); + expect(collectionName).to.be.equal('cats'); + }); + + suite('when not connected', function () { + let connectWithConnectionIdStub; + let changeActiveConnectionStub; + let getSavedConnectionsStub; + + beforeEach(function () { + connectWithConnectionIdStub = sinon.stub( + testParticipantController._connectionController, + 'connectWithConnectionId' + ); + changeActiveConnectionStub = sinon.stub( + testParticipantController._connectionController, + 'changeActiveConnection' + ); + sinon.replace( + testParticipantController._connectionController, + 'getActiveDataService', + () => null + ); + sinon.replace( + testParticipantController._storageController, + 'get', + sinon.fake.returns(true) + ); + getSavedConnectionsStub = sinon.stub(); + sinon.replace( + testParticipantController._connectionController, + 'getSavedConnections', + getSavedConnectionsStub + ); + }); + + test('asks to connect', async function () { + getSavedConnectionsStub.returns([loadedConnection]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + const connectMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(connectMessage).to.include( + "Looks like you aren't currently connected, first let's get you connected to the cluster we'd like to create this query to run against." + ); + const listConnectionsMessage = chatStreamStub.markdown.getCall(1).args[0]; + const expectedContent = encodeStringify({ id: 'id', command: '/query' }); + expect(listConnectionsMessage.value).to.include( + `- [localhost](command:mdb.connectWithParticipant?${expectedContent})` + ); + const showMoreMessage = chatStreamStub.markdown.getCall(2).args[0]; + expect(showMoreMessage.value).to.include( + `- [Show more](command:mdb.connectWithParticipant?${encodeStringify({ + command: '/query', + })})` + ); + expect(chatResult?.metadata?.chatId.length).to.equal(testChatId.length); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askToConnect', + chatId: undefined, + }); + }); + + test('shows only 10 connections with the show more option', async function () { + const connections: LoadedConnection[] = []; + for (let i = 0; i < 11; i++) { + connections.push({ + ...loadedConnection, + id: `${loadedConnection.id}${i}`, + name: `${loadedConnection.name}${i}`, + }); + } + getSavedConnectionsStub.returns(connections); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + const connectMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(connectMessage).to.include( + "Looks like you aren't currently connected, first let's get you connected to the cluster we'd like to create this query to run against." + ); + const listConnectionsMessage = chatStreamStub.markdown.getCall(1).args[0]; + const expectedContent = encodeStringify({ id: 'id0', command: '/query' }); + expect(listConnectionsMessage.value).to.include( + `- [localhost0](command:mdb.connectWithParticipant?${expectedContent})` + ); + const showMoreMessage = chatStreamStub.markdown.getCall(11).args[0]; + expect(showMoreMessage.value).to.include( + `- [Show more](command:mdb.connectWithParticipant?${encodeStringify({ + command: '/query', + })})` + ); + expect(chatStreamStub.markdown.callCount).to.be.eql(12); + expect(chatResult?.metadata?.chatId.length).to.equal(testChatId.length); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askToConnect', + chatId: undefined, + }); + }); + + test('handles empty connection name', async function () { + getSavedConnectionsStub.returns([loadedConnection]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + + chatRequestMock.prompt = ''; + await invokeChatHandler(chatRequestMock); + + const emptyMessage = chatStreamStub.markdown.getCall(3).args[0]; + expect(emptyMessage).to.include( + "Looks like you aren't currently connected, first let's get you connected to the cluster we'd like to create this query to run against" + ); + const listConnectionsMessage = chatStreamStub.markdown.getCall(4).args[0]; + const expectedContent = encodeStringify({ id: 'id', command: '/query' }); + expect(listConnectionsMessage.value).to.include( + `- [localhost](command:mdb.connectWithParticipant?${expectedContent})` + ); + const showMoreMessage = chatStreamStub.markdown.getCall(5).args[0]; + expect(showMoreMessage.value).to.include( + `- [Show more](command:mdb.connectWithParticipant?${encodeStringify({ + command: '/query', + })})` + ); + expect(chatResult?.metadata?.chatId.length).to.equal(testChatId.length); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askToConnect', + chatId: undefined, + }); + }); + + test('calls connect by id for an existing connection', async function () { + await testParticipantController.connectWithParticipant({ + id: '123', + }); + expect(connectWithConnectionIdStub).to.have.been.calledWithExactly('123'); + }); + + test('calls connect with uri for a new connection', async function () { + await testParticipantController.connectWithParticipant({}); + expect(changeActiveConnectionStub).to.have.been.called; + }); + }); + + suite('when connected', function () { + let sampleStub; + + beforeEach(function () { + sampleStub = sinon.stub(); + sinon.replace( + testParticipantController._connectionController, + 'getActiveDataService', + () => + ({ + listDatabases: () => + Promise.resolve([ + { name: 'dbOne' }, + { name: 'customer' }, + { name: 'inventory' }, + { name: 'sales' }, + { name: 'employee' }, + { name: 'financialReports' }, + { name: 'productCatalog' }, + { name: 'projectTracker' }, + { name: 'user' }, + { name: 'analytics' }, + { name: '123' }, + ]), + listCollections: () => + Promise.resolve([ + { name: 'collOne' }, + { name: 'notifications' }, + { name: 'products' }, + { name: 'orders' }, + { name: 'categories' }, + { name: 'invoices' }, + { name: 'transactions' }, + { name: 'logs' }, + { name: 'messages' }, + { name: 'sessions' }, + { name: 'feedback' }, + ]), + getMongoClientConnectionOptions: () => ({ + url: TEST_DATABASE_URI, + options: {}, + }), + sample: sampleStub, + once: sinon.stub(), + } as unknown as DataService) + ); + }); + + suite('when has not been shown a welcome message yet', function () { + beforeEach(function () { + sinon.replace( + testParticipantController._storageController, + 'get', + sinon.fake.returns(false) + ); + }); + + test('prints a welcome message to chat', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const welcomeMessage = chatStreamStub.markdown.firstCall.args[0]; + expect(welcomeMessage).to.include('Welcome to MongoDB Participant!'); + + // Once to report welcome screen shown, second time to track the user prompt + expect(telemetryTrackStub).to.have.been.calledTwice; + expect(telemetryTrackStub.firstCall.args[0]).to.equal( + TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN + ); + expect(telemetryTrackStub.firstCall.args[1]).to.be.undefined; + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + expectedInternalPurpose: 'namespace', + }); + }); + }); + + suite('when has been shown a welcome message already', function () { + beforeEach(function () { + sinon.replace( + testParticipantController._storageController, + 'get', + sinon.fake.returns(true) + ); + }); + + afterEach(function () { + // Ensure welcome message is not shown again + const welcomeMessages = chatStreamStub.markdown + .getCalls() + .map((call) => call.args[0]) + .filter( + (message) => + typeof message === 'string' && + message.includes('Welcome to MongoDB Participant!') + ); + expect(welcomeMessages).to.be.empty; + + // Ensure we haven't reported the welcome screen to telemetry + const telemetryEvents = telemetryTrackStub + .getCalls() + .map((call) => call.args[0]) + .filter( + (arg) => arg === TelemetryEventTypes.PARTICIPANT_WELCOME_SHOWN + ); + + expect(telemetryEvents).to.be.empty; + }); + + suite('generic command', function () { + suite('when the intent is recognized', function () { + beforeEach(function () { + sendRequestStub.onCall(0).resolves({ + text: ['Schema'], + }); + }); + + test('routes to the appropriate handler', async function () { + const chatRequestMock = { + prompt: + 'what is the shape of the documents in the pineapple collection?', + command: undefined, + references: [], + }; + const res = await invokeChatHandler(chatRequestMock); + + expect(sendRequestStub).to.have.been.calledTwice; + const intentRequest = sendRequestStub.firstCall.args[0]; + expect(intentRequest).to.have.length(2); + expect(intentRequest[0].content).to.include( + 'Your task is to help guide a conversation with a user to the correct handler.' + ); + expect(intentRequest[1].content).to.equal( + 'what is the shape of the documents in the pineapple collection?' + ); + const genericRequest = sendRequestStub.secondCall.args[0]; + expect(genericRequest).to.have.length(2); + expect(genericRequest[0].content).to.include( + 'Parse all user messages to find a database name and a collection name.' + ); + expect(genericRequest[1].content).to.equal( + 'what is the shape of the documents in the pineapple collection?' + ); + + expect(res?.metadata.intent).to.equal('askForNamespace'); + }); + }); + + test('default handler asks for intent and shows code run actions', async function () { + const chatRequestMock = { + prompt: 'how to find documents in my collection?', + command: undefined, + references: [], + }; + const res = await invokeChatHandler(chatRequestMock); + + expect(sendRequestStub).to.have.been.calledTwice; + const intentRequest = sendRequestStub.firstCall.args[0]; + expect(intentRequest).to.have.length(2); + expect(intentRequest[0].content).to.include( + 'Your task is to help guide a conversation with a user to the correct handler.' + ); + expect(intentRequest[1].content).to.equal( + 'how to find documents in my collection?' + ); + const genericRequest = sendRequestStub.secondCall.args[0]; + expect(genericRequest).to.have.length(2); + expect(genericRequest[0].content).to.include( + 'Your task is to help the user with MongoDB related questions.' + ); + expect(genericRequest[1].content).to.equal( + 'how to find documents in my collection?' + ); + + expect(res?.metadata.intent).to.equal('generic'); + expect(chatStreamStub?.button.getCall(0).args[0]).to.deep.equal({ + command: 'mdb.runParticipantCode', + title: '▶️ Run', + arguments: [ + { + runnableContent: + "use('dbOne');\ndb.getCollection('collOne').find({ name: 'example' });", + }, + ], + }); + + assertCommandTelemetry('generic', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'intent', + }); + + assertCommandTelemetry('generic', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('generic', { + callIndex: 2, + hasRunnableContent: true, + }); + }); + }); + + suite('query command', function () { + suite('known namespace from running namespace LLM', function () { + beforeEach(function () { + sendRequestStub.onCall(0).resolves({ + text: ['DATABASE_NAME: dbOne\n', 'COLLECTION_NAME: collOne\n`'], + }); + }); + + test('generates a query', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(chatStreamStub?.button.getCall(0).args[0]).to.deep.equal({ + command: 'mdb.runParticipantCode', + title: '▶️ Run', + arguments: [ + { + runnableContent: + "use('dbOne');\ndb.getCollection('collOne').find({ name: 'example' });", + }, + ], + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + + test('includes a collection schema', async function () { + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203660'), + field: { + stringField: + 'There was a house cat who finally got the chance to do what it had always wanted to do.', + arrayField: [new Int32('1')], + }, + }, + ]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.include( + 'Collection schema: _id: ObjectId\n' + + 'field.stringField: String\n' + + 'field.arrayField: Array\n' + ); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + + suite('useSampleDocsInCopilot setting is true', function () { + beforeEach(async () => { + await vscode.workspace + .getConfiguration('mdb') + .update('useSampleDocsInCopilot', true); + }); + + afterEach(async () => { + await vscode.workspace + .getConfiguration('mdb') + .update('useSampleDocsInCopilot', false); + }); + + test('includes 3 sample documents as an array', async function () { + countTokensStub.resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK); + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203661'), + field: { + stringField: 'Text 1', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203662'), + field: { + stringField: 'Text 2', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203663'), + field: { + stringField: 'Text 3', + }, + }, + ]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.include( + 'Sample documents: [\n' + + ' {\n' + + " _id: ObjectId('63ed1d522d8573fa5c203661'),\n" + + ' field: {\n' + + " stringField: 'Text 1'\n" + + ' }\n' + + ' },\n' + + ' {\n' + + " _id: ObjectId('63ed1d522d8573fa5c203662'),\n" + + ' field: {\n' + + " stringField: 'Text 2'\n" + + ' }\n' + + ' },\n' + + ' {\n' + + " _id: ObjectId('63ed1d522d8573fa5c203663'),\n" + + ' field: {\n' + + " stringField: 'Text 3'\n" + + ' }\n' + + ' }\n' + + ']\n' + ); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + + test('includes 1 sample document as an object', async function () { + countTokensStub.resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK); + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203660'), + field: { + stringField: + 'There was a house cat who finally got the chance to do what it had always wanted to do.', + arrayField: [ + new Int32('1'), + new Int32('2'), + new Int32('3'), + new Int32('4'), + new Int32('5'), + new Int32('6'), + new Int32('7'), + new Int32('8'), + new Int32('9'), + ], + }, + }, + ]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.include( + 'Sample document: {\n' + + " _id: ObjectId('63ed1d522d8573fa5c203660'),\n" + + ' field: {\n' + + " stringField: 'There was a house ca',\n" + + ' arrayField: [\n' + + " NumberInt('1'),\n" + + " NumberInt('2'),\n" + + " NumberInt('3')\n" + + ' ]\n' + + ' }\n' + + '}\n' + ); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + + test('includes 1 sample documents when 3 make prompt too long', async function () { + countTokensStub + .onCall(0) + .resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK + 1); + countTokensStub.onCall(1).resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK); + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203661'), + field: { + stringField: 'Text 1', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203662'), + field: { + stringField: 'Text 2', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203663'), + field: { + stringField: 'Text 3', + }, + }, + ]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.include( + 'Sample document: {\n' + + " _id: ObjectId('63ed1d522d8573fa5c203661'),\n" + + ' field: {\n' + + " stringField: 'Text 1'\n" + + ' }\n' + + '}\n' + ); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + expectSampleDocs: true, + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + + test('does not include sample documents when even 1 makes prompt too long', async function () { + countTokensStub + .onCall(0) + .resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK + 1); + countTokensStub + .onCall(1) + .resolves(MAX_TOTAL_PROMPT_LENGTH_MOCK + 1); + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203661'), + field: { + stringField: 'Text 1', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203662'), + field: { + stringField: 'Text 2', + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203663'), + field: { + stringField: 'Text 3', + }, + }, + ]); + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.not.include('Sample documents'); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + }); + + suite('useSampleDocsInCopilot setting is false', function () { + test('does not include sample documents', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[1].content).to.not.include('Sample documents'); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('query', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('query', { + callIndex: 2, + hasRunnableContent: true, + foundNamespace: true, + }); + }); + }); + }); + + suite('unknown namespace', function () { + test('asks for a namespace and generates a query', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + const askForDBMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(askForDBMessage).to.include( + 'What is the name of the database you would like this query to run against?' + ); + const listDBsMessage = chatStreamStub.markdown.getCall(1).args[0]; + const expectedContent = encodeStringify({ + command: '/query', + chatId: testChatId, + databaseName: 'dbOne', + }); + expect(listDBsMessage.value).to.include( + `- [dbOne](command:mdb.selectDatabaseWithParticipant?${expectedContent})` + ); + const showMoreDBsMessage = + chatStreamStub.markdown.getCall(11).args[0]; + expect(showMoreDBsMessage.value).to.include( + `- [Show more](command:mdb.selectDatabaseWithParticipant?${encodeStringify( + { command: '/query', chatId: testChatId } + )})` + ); + expect(chatStreamStub.markdown.callCount).to.be.eql(12); + const firstChatId = chatResult?.metadata?.chatId; + expect(chatResult?.metadata?.chatId.length).to.equal( + testChatId.length + ); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askForNamespace', + collectionName: undefined, + databaseName: undefined, + chatId: undefined, + }); + + chatRequestMock.prompt = 'dbOne'; + sendRequestStub.onCall(1).resolves({ + text: ['DATABASE_NAME: dbOne\n'], + }); + + chatContextStub = { + history: [ + { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + } as vscode.ChatRequestTurn, + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'What is the name of the database you would like this query to run against?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + chatId: firstChatId, + }, + }, + } as vscode.ChatResponseTurn + ), + ], + }; + + const chatResult2 = await invokeChatHandler(chatRequestMock); + + const askForCollMessage = + chatStreamStub.markdown.getCall(12).args[0]; + expect(askForCollMessage).to.include( + 'Which collection would you like to use within dbOne?' + ); + const listCollsMessage = + chatStreamStub.markdown.getCall(13).args[0]; + const expectedCollsContent = encodeStringify({ + command: '/query', + chatId: testChatId, + databaseName: 'dbOne', + collectionName: 'collOne', + }); + expect(listCollsMessage.value).to.include( + `- [collOne](command:mdb.selectCollectionWithParticipant?${expectedCollsContent})` + ); + const showMoreCollsMessage = + chatStreamStub.markdown.getCall(23).args[0]; + expect(showMoreCollsMessage.value).to.include( + `- [Show more](command:mdb.selectCollectionWithParticipant?${encodeStringify( + { + command: '/query', + chatId: testChatId, + databaseName: 'dbOne', + } + )})` + ); + expect(chatStreamStub.markdown.callCount).to.be.eql(24); + expect(chatResult2?.metadata?.chatId).to.equal(firstChatId); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askForNamespace', + collectionName: undefined, + databaseName: undefined, + chatId: undefined, + }); + + chatRequestMock.prompt = 'collOne'; + sendRequestStub.onCall(2).resolves({ + text: ['DATABASE_NAME: dbOne\n', 'COLLECTION_NAME: collOne\n`'], + }); + chatContextStub = { + history: [ + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'Which database would you like to query within this database?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + }, + }, + } + ), + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'dbOne', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'Which collection would you like to query within dbOne?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + databaseName: 'dbOne', + collectionName: 'collOne', + chatId: firstChatId, + }, + }, + } + ), + ], + }; + await invokeChatHandler(chatRequestMock); + + expect(chatStreamStub?.button.callCount).to.equal(2); + expect(chatStreamStub?.button.getCall(0).args[0]).to.deep.equal({ + command: 'mdb.runParticipantCode', + title: '▶️ Run', + arguments: [ + { + runnableContent: + "use('dbOne');\ndb.getCollection('collOne').find({ name: 'example' });", + }, + ], + }); + expect(chatStreamStub?.button.getCall(1).args[0]).to.deep.equal({ + command: 'mdb.openParticipantCodeInPlayground', + title: 'Open in playground', + arguments: [ + { + runnableContent: + "use('dbOne');\ndb.getCollection('collOne').find({ name: 'example' });", + }, + ], + }); + }); + + test('handles empty database name', async function () { + const chatRequestMock = { + prompt: '', + command: 'query', + references: [], + }; + chatContextStub = { + history: [ + { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + } as vscode.ChatRequestTurn, + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'What is the name of the database you would like this query to run against?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + chatId: 'pineapple', + }, + }, + } as vscode.ChatResponseTurn + ), + ], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + + const emptyMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(emptyMessage).to.include( + 'Please select a database by either clicking on an item in the list or typing the name manually in the chat.' + ); + const listDBsMessage = chatStreamStub.markdown.getCall(1).args[0]; + expect(listDBsMessage.value).to.include( + `- [dbOne](command:mdb.selectDatabaseWithParticipant?${encodeStringify( + { + command: '/query', + chatId: 'pineapple', + databaseName: 'dbOne', + } + )})` + ); + const showMoreDBsMessage = + chatStreamStub.markdown.getCall(11).args[0]; + expect(showMoreDBsMessage.value).to.include( + `- [Show more](command:mdb.selectDatabaseWithParticipant?${encodeStringify( + { + command: '/query', + chatId: 'pineapple', + } + )})` + ); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askForNamespace', + collectionName: undefined, + databaseName: undefined, + chatId: undefined, + }); + }); + + test('handles empty collection name', async function () { + const chatRequestMock = { + prompt: '', + command: 'query', + references: [], + }; + chatContextStub = { + history: [ + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'find all docs by a name example', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'Which database would you like to query within this database?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + }, + }, + } + ), + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'dbOne', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign( + Object.create(vscode.ChatResponseTurn.prototype), + { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: + 'Which collection would you like to query within dbOne?', + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askForNamespace', + databaseName: 'dbOne', + collectionName: undefined, + chatId: 'pineapple', + }, + }, + } + ), + ], + }; + const chatResult = await invokeChatHandler(chatRequestMock); + + const emptyMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(emptyMessage).to.include( + 'Please select a collection by either clicking on an item in the list or typing the name manually in the chat.' + ); + const listCollsMessage = chatStreamStub.markdown.getCall(1).args[0]; + expect(listCollsMessage.value).to.include( + `- [collOne](command:mdb.selectCollectionWithParticipant?${encodeStringify( + { + command: '/query', + chatId: 'pineapple', + databaseName: 'dbOne', + collectionName: 'collOne', + } + )})` + ); + const showMoreCollsMessage = + chatStreamStub.markdown.getCall(11).args[0]; + expect(showMoreCollsMessage.value).to.include( + `- [Show more](command:mdb.selectCollectionWithParticipant?${encodeStringify( + { + command: '/query', + chatId: 'pineapple', + databaseName: 'dbOne', + } + )})` + ); + expect({ + ...chatResult?.metadata, + chatId: undefined, + }).to.deep.equal({ + intent: 'askForNamespace', + collectionName: undefined, + databaseName: 'dbOne', + chatId: undefined, + }); + }); + }); + }); + + suite('schema command', function () { + suite('no namespace provided', function () { + beforeEach(function () { + sendRequestStub.onCall(0).resolves({ + text: ['none'], + }); + }); + + test('without a prompt it asks for the database name without pinging ai', async function () { + const chatRequestMock = { + prompt: '', + command: 'schema', + references: [], + }; + await invokeChatHandler(chatRequestMock); + + expect(sendRequestStub.called).to.be.false; + const askForDBMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(askForDBMessage).to.include( + 'What is the name of the database you would like to run against?' + ); + }); + + test('with a prompt it asks the ai for the namespace', async function () { + const chatRequestMock = { + prompt: 'pineapple', + command: 'schema', + references: [], + }; + await invokeChatHandler(chatRequestMock); + + expect(sendRequestStub.calledOnce).to.be.true; + expect(sendRequestStub.firstCall.args[0][0].content).to.include( + 'Parse all user messages to find a database name and a collection name.' + ); + + const askForDBMessage = chatStreamStub.markdown.getCall(0).args[0]; + expect(askForDBMessage).to.include( + 'What is the name of the database you would like to run against?' + ); + }); + }); + + suite( + 'with a prompt and a known namespace from running namespace LLM', + function () { + beforeEach(function () { + sendRequestStub.onCall(0).resolves({ + text: ['DATABASE_NAME: dbOne\n', 'COLLECTION_NAME: collOne\n`'], + }); + }); + + test('shows a button to view the json output', async function () { + const chatRequestMock = { + prompt: 'what is my schema', + command: 'schema', + references: [], + }; + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203660'), + }, + ]); + await invokeChatHandler(chatRequestMock); + const expectedSchema = `{ + "count": 1, + "fields": [ + { + "name": "_id", + "path": [ + "_id" + ], + "count": 1, + "type": "ObjectId", + "probability": 1, + "hasDuplicates": false, + "types": [ + { + "name": "ObjectId", + "path": [ + "_id" + ], + "count": 1, + "probability": 1, + "bsonType": "ObjectId" + } + ] + } + ] +}`; + expect(chatStreamStub?.button.getCall(0).args[0]).to.deep.equal({ + command: 'mdb.participantViewRawSchemaOutput', + title: 'Open JSON Output', + arguments: [ + { + schema: expectedSchema, + }, + ], + }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('schema', { + callIndex: 2, + hasCTA: true, + foundNamespace: true, + }); + }); + + test("includes the collection's schema in the request", async function () { + sampleStub.resolves([ + { + _id: new ObjectId('63ed1d522d8573fa5c203660'), + field: { + stringField: + 'There was a house cat who finally got the chance to do what it had always wanted to do.', + arrayField: [new Int32('1')], + }, + }, + { + _id: new ObjectId('63ed1d522d8573fa5c203660'), + field: { + stringField: 'Pineapple.', + arrayField: [new Int32('166')], + }, + }, + ]); + const chatRequestMock = { + prompt: 'what is my schema', + command: 'schema', + references: [], + }; + await invokeChatHandler(chatRequestMock); + const messages = sendRequestStub.secondCall.args[0]; + expect(messages[0].content).to.include( + 'Amount of documents sampled: 2' + ); + expect(messages[1].content).to.include( + `Database name: dbOne +Collection name: collOne +Schema: +{ + "count": 2, + "fields": [` + ); + expect(messages[1].content).to.include(`"name": "arrayField", + "path": [ + "field", + "arrayField" + ],`); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 1, + }); + + assertResponseTelemetry('schema', { + callIndex: 2, + hasCTA: true, + foundNamespace: true, + }); + }); + + test('prints a message when no documents are found', async function () { + sampleStub.resolves([]); + const chatRequestMock = { + prompt: 'what is my schema', + command: 'schema', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(chatStreamStub?.markdown.getCall(0).args[0]).to.include( + 'Unable to generate a schema from the collection, no documents found.' + ); + + assertCommandTelemetry('schema', chatRequestMock, { + callIndex: 0, + expectedInternalPurpose: 'namespace', + }); + }); + } + ); + }); + + suite('docs command', function () { + const initialFetch = global.fetch; + let fetchStub: sinon.SinonStub; + + beforeEach(function () { + sendRequestStub.onCall(0).resolves({ + text: ['connection info'], + }); + }); + + afterEach(function () { + global.fetch = initialFetch; + sinon.restore(); + }); + + test('uses docs chatbot result if available', async function () { + fetchStub = sinon.stub().resolves({ + status: 200, + ok: true, + json: () => + Promise.resolve({ + _id: '650b4b260f975ef031016c8a', + content: + 'To connect to MongoDB using mongosh, you can follow these steps', + }), + }); + global.fetch = fetchStub; + const chatRequestMock = { + prompt: 'how to connect to mongodb', + command: 'docs', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(fetchStub).to.have.been.called; + expect(sendRequestStub).to.have.not.been.called; + + assertResponseTelemetry('docs/chatbot', { + callIndex: 0, + }); + }); + + test('falls back to the copilot model when docs chatbot result is not available', async function () { + fetchStub = sinon.stub().resolves({ + status: 500, + ok: false, + statusText: 'Internal Server Error', + json: () => Promise.reject(new Error('invalid json')), + }); + global.fetch = fetchStub; + const chatRequestMock = { + prompt: 'how to connect to mongodb', + command: 'docs', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(sendRequestStub).to.have.been.called; + + // Expect the error to be reported through the telemetry service + expect( + telemetryTrackStub.getCalls() + ).to.have.length.greaterThanOrEqual(2); + expect(telemetryTrackStub.firstCall.args[0]).to.equal( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + ); + + const properties = telemetryTrackStub.firstCall.args[1]; + expect(properties.command).to.equal('docs'); + expect(properties.error_name).to.equal('Docs Chatbot API Issue'); + + assertResponseTelemetry('docs/copilot', { + callIndex: 2, + hasCTA: true, + }); + }); + }); + }); + }); + + suite('prompt builders', function () { + test('generic', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + }; + const { messages, stats } = await Prompts.generic.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + connectionNames: [], + }); + + expect(messages).to.have.lengthOf(2); + expect(messages[0].role).to.equal( + vscode.LanguageModelChatMessageRole.Assistant + ); + expect(messages[1].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + + expect(stats.command).to.equal('generic'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); + }); + + test('query', async function () { + const chatRequestMock = { + prompt: + 'how do I find the number of people whose name starts with "P"?', + command: 'query', + }; + + chatContextStub = { + history: [ + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: 'give me the count of all people in the prod database', + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + ], + }; + const { messages, stats } = await Prompts.query.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + collectionName: 'people', + connectionNames: ['localhost', 'atlas'], + databaseName: 'prod', + sampleDocuments: [ + { + _id: new ObjectId(), + name: 'Peter', + }, + { + _id: new ObjectId(), + name: 'John', + }, + ], + schema: ` + { + _id: ObjectId, + name: String + } + `, + }); + + expect(messages).to.have.lengthOf(3); + + // Assistant prompt + expect(messages[0].role).to.equal( + vscode.LanguageModelChatMessageRole.Assistant + ); + + // History + expect(messages[1].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + expect(messages[1].content).to.equal( + 'give me the count of all people in the prod database' + ); + + // Actual user prompt + expect(messages[2].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + + expect(stats.command).to.equal('query'); + expect(stats.has_sample_documents).to.be.true; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + + messages[1].content.length + + messages[2].content.length + ); + + // The length of the user prompt length should be taken from the prompt supplied + // by the user, even if we enhance it with sample docs and schema. + expect(stats.user_input_length).to.be.lessThan( + messages[2].content.length + ); + }); + + test('schema', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'schema', + }; + + const databaseName = 'dbOne'; + const collectionName = 'collOne'; + const schema = ` + { + _id: ObjectId, + name: String + } + `; + const { messages, stats } = await Prompts.schema.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + amountOfDocumentsSampled: 3, + collectionName, + databaseName, + schema, + connectionNames: [], + }); + + expect(messages).to.have.lengthOf(2); + expect(messages[0].role).to.equal( + vscode.LanguageModelChatMessageRole.Assistant + ); + expect(messages[0].content).to.include('Amount of documents sampled: 3'); + + expect(messages[1].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + expect(messages[1].content).to.include(databaseName); + expect(messages[1].content).to.include(collectionName); + expect(messages[1].content).to.include(schema); + + expect(stats.command).to.equal('schema'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); + }); + + test('namespace', async function () { + const chatRequestMock = { + prompt: 'find all docs by a name example', + command: 'query', + }; + const { messages, stats } = await Prompts.namespace.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + connectionNames: [], + }); + + expect(messages).to.have.lengthOf(2); + expect(messages[0].role).to.equal( + vscode.LanguageModelChatMessageRole.Assistant + ); + expect(messages[1].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + + expect(stats.command).to.equal('query'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(chatRequestMock.prompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); + }); + + test('removes askForConnect messages from history', async function () { + // The user is responding to an `askToConnect` message, so the prompt is just the + // name of the connection + const chatRequestMock = { + prompt: 'localhost', + command: 'query', + }; + + // This is the prompt of the user prior to us asking them to connect + const expectedPrompt = + 'give me the count of all people in the prod database'; + + chatContextStub = { + history: [ + Object.assign(Object.create(vscode.ChatRequestTurn.prototype), { + prompt: expectedPrompt, + command: 'query', + references: [], + participant: CHAT_PARTICIPANT_ID, + }), + Object.assign(Object.create(vscode.ChatResponseTurn.prototype), { + participant: CHAT_PARTICIPANT_ID, + response: [ + { + value: { + value: `Looks like you aren't currently connected, first let's get you connected to the cluster we'd like to create this query to run against. + + ${createMarkdownLink({ + commandId: EXTENSION_COMMANDS.CONNECT_WITH_PARTICIPANT, + name: 'localhost', + data: {}, + })} + ${createMarkdownLink({ + commandId: EXTENSION_COMMANDS.CONNECT_WITH_PARTICIPANT, + name: 'atlas', + data: {}, + })}`, + } as vscode.MarkdownString, + }, + ], + command: 'query', + result: { + metadata: { + intent: 'askToConnect', + chatId: 'abc', + }, + }, + }), + ], + }; + + const { messages, stats } = await Prompts.query.buildMessages({ + context: chatContextStub, + request: chatRequestMock, + collectionName: 'people', + connectionNames: ['localhost', 'atlas'], + databaseName: 'prod', + sampleDocuments: [], + }); + + expect(messages.length).to.equal(2); + expect(messages[0].role).to.equal( + vscode.LanguageModelChatMessageRole.Assistant + ); + + // We don't expect history because we're removing the askForConnect message as well + // as the user response to it. Therefore the actual user prompt should be the first + // message that we supplied in the history. + expect(messages[1].role).to.equal( + vscode.LanguageModelChatMessageRole.User + ); + expect(messages[1].content).to.contain(expectedPrompt); + + expect(stats.command).to.equal('query'); + expect(stats.has_sample_documents).to.be.false; + expect(stats.user_input_length).to.equal(expectedPrompt.length); + expect(stats.total_message_length).to.equal( + messages[0].content.length + messages[1].content.length + ); + + // The prompt builder may add extra info, but we're only reporting the actual user input + expect(stats.user_input_length).to.be.lessThan( + messages[1].content.length + ); + }); + }); + + suite('telemetry', function () { + test('reports positive user feedback', async function () { + await testParticipantController.handleUserFeedback({ + kind: vscode.ChatResultFeedbackKind.Helpful, + result: { + metadata: { + intent: 'askToConnect', + responseContent: '```creditCardNumber: 1234-5678-9012-3456```', + }, + }, + }); + + sinon.assert.calledOnce(telemetryTrackStub); + expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( + 'Participant Feedback' + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.feedback).to.be.equal('positive'); + expect(properties.reason).to.be.undefined; + expect(properties.response_type).to.be.equal('askToConnect'); + + // Ensure we're not leaking the response content into the telemetry payload + expect(JSON.stringify(properties)) + .to.not.include('creditCardNumber') + .and.not.include('1234-5678-9012-3456'); + }); + + test('reports negative user feedback', async function () { + await testParticipantController.handleUserFeedback({ + kind: vscode.ChatResultFeedbackKind.Unhelpful, + result: { + metadata: { + intent: 'query', + responseContent: 'SSN: 123456789', + }, + }, + unhelpfulReason: 'incompleteCode', + } as vscode.ChatResultFeedback); + + sinon.assert.calledOnce(telemetryTrackStub); + expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( + 'Participant Feedback' + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.feedback).to.be.equal('negative'); + expect(properties.reason).to.be.equal('incompleteCode'); + expect(properties.response_type).to.be.equal('query'); + + // Ensure we're not leaking the response content into the telemetry payload + expect(JSON.stringify(properties)) + .to.not.include('SSN') + .and.not.include('123456789'); + }); + + test('reports error', function () { + const err = Error('Filtered by Responsible AI Service'); + expect(() => testParticipantController.handleError(err, 'query')).throws( + 'Filtered by Responsible AI Service' + ); + sinon.assert.calledOnce(telemetryTrackStub); + + expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( + 'Participant Response Failed' + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.command).to.be.equal('query'); + expect(properties.error_code).to.be.undefined; + expect(properties.error_name).to.be.equal( + 'Filtered by Responsible AI Service' + ); + }); + + test('reports nested error', function () { + const err = new Error('Parent error'); + err.cause = Error('This message is flagged as off topic: off_topic.'); + expect(() => testParticipantController.handleError(err, 'docs')).throws( + 'off_topic' + ); + sinon.assert.calledOnce(telemetryTrackStub); + + expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( + 'Participant Response Failed' + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.command).to.be.equal('docs'); + expect(properties.error_code).to.be.undefined; + expect(properties.error_name).to.be.equal('Chat Model Off Topic'); + }); + + test('Reports error code when available', function () { + // eslint-disable-next-line new-cap + const err = vscode.LanguageModelError.NotFound('Model not found'); + expect(() => testParticipantController.handleError(err, 'schema')).throws( + 'Model not found' + ); + sinon.assert.calledOnce(telemetryTrackStub); + + expect(telemetryTrackStub.lastCall.args[0]).to.be.equal( + 'Participant Response Failed' + ); + + const properties = telemetryTrackStub.lastCall.args[1]; + expect(properties.command).to.be.equal('schema'); + expect(properties.error_code).to.be.equal('NotFound'); + expect(properties.error_name).to.be.equal('Other'); + }); + }); +}); diff --git a/src/test/suite/participant/streamParsing.test.ts b/src/test/suite/participant/streamParsing.test.ts new file mode 100644 index 000000000..66208ecdd --- /dev/null +++ b/src/test/suite/participant/streamParsing.test.ts @@ -0,0 +1,219 @@ +import { beforeEach } from 'mocha'; +import { expect } from 'chai'; + +import { processStreamWithIdentifiers } from '../../../participant/streamParsing'; +import { asyncIterableFromArray } from './asyncIterableFromArray'; + +const defaultCodeBlockIdentifier = { + start: '```', + end: '```', +}; + +suite('processStreamWithIdentifiers', () => { + let fragmentsProcessed: string[] = []; + let identifiersStreamed: string[] = []; + + const processStreamFragment = (fragment: string): void => { + fragmentsProcessed.push(fragment); + }; + + const onStreamIdentifier = (content: string): void => { + identifiersStreamed.push(content); + }; + + beforeEach(function () { + fragmentsProcessed = []; + identifiersStreamed = []; + }); + + test('empty', async () => { + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable: asyncIterableFromArray([]), + identifier: defaultCodeBlockIdentifier, + }); + + expect(fragmentsProcessed).to.be.empty; + expect(identifiersStreamed).to.be.empty; + }); + + test('input with no code block', async () => { + const inputText = 'This is some sample text without code blocks.'; + const inputFragments = inputText.match(/.{1,5}/g) || []; + const inputIterable = asyncIterableFromArray(inputFragments); + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier: defaultCodeBlockIdentifier, + }); + + expect(fragmentsProcessed.join('')).to.equal(inputText); + expect(identifiersStreamed).to.be.empty; + }); + + test('one code block with fragment sizes 2', async () => { + const inputText = '```javascript\npineapple\n```\nMore text.'; + const inputFragments: string[] = []; + let index = 0; + const fragmentSize = 2; + while (index < inputText.length) { + const fragment = inputText.substr(index, fragmentSize); + inputFragments.push(fragment); + index += fragmentSize; + } + + const inputIterable = asyncIterableFromArray(inputFragments); + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier: { + start: '```javascript', + end: '```', + }, + }); + + expect(fragmentsProcessed.join('')).to.equal(inputText); + expect(identifiersStreamed).to.have.lengthOf(1); + expect(identifiersStreamed[0]).to.equal('\npineapple\n'); + }); + + test('multiple code blocks', async () => { + const inputText = + 'Text before code.\n```\ncode1\n```\nText between code.\n```\ncode2\n```\nText after code.'; + const inputFragments = inputText.split(''); + + const inputIterable = asyncIterableFromArray(inputFragments); + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier: defaultCodeBlockIdentifier, + }); + + expect(fragmentsProcessed.join('')).to.equal(inputText); + expect(identifiersStreamed).to.deep.equal(['\ncode1\n', '\ncode2\n']); + }); + + test('unfinished code block', async () => { + const inputText = + 'Text before code.\n```\ncode content without end identifier.'; + const inputFragments = inputText.split(''); + + const inputIterable = asyncIterableFromArray(inputFragments); + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier: defaultCodeBlockIdentifier, + }); + + expect(fragmentsProcessed.join('')).to.equal(inputText); + expect(identifiersStreamed).to.be.empty; + }); + + test('code block identifier is a fragment', async () => { + const inputFragments = [ + 'Text before code.\n', + '```js', + '\ncode content\n', + '```', + '```js', + '\npineapple\n', + '```', + '\nText after code.', + ]; + + const inputIterable = asyncIterableFromArray(inputFragments); + + const identifier = { start: '```js', end: '```' }; + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier, + }); + + expect(fragmentsProcessed.join('')).to.deep.equal(inputFragments.join('')); + + expect(identifiersStreamed).to.deep.equal([ + '\ncode content\n', + '\npineapple\n', + ]); + }); + + test('code block identifier split between fragments', async () => { + const inputFragments = [ + 'Text before code.\n`', + '``j', + 's\ncode content\n`', + '``', + '\nText after code.', + ]; + + const inputIterable = asyncIterableFromArray(inputFragments); + + const identifier = { start: '```js', end: '```' }; + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier, + }); + + expect(fragmentsProcessed.join('')).to.deep.equal(inputFragments.join('')); + + expect(identifiersStreamed).to.deep.equal(['\ncode content\n']); + }); + + test('fragments containing multiple code blocks', async () => { + const inputFragments = [ + 'Text before code.\n```', + 'js\ncode1\n```', + '\nText', + ' between code.\n``', + '`js\ncode2\n``', + '`\nText after code.', + ]; + + const inputIterable = asyncIterableFromArray(inputFragments); + const identifier = { start: '```js', end: '```' }; + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier, + }); + + expect(fragmentsProcessed.join('')).to.deep.equal(inputFragments.join('')); + expect(identifiersStreamed).to.deep.equal(['\ncode1\n', '\ncode2\n']); + }); + + test('one fragment containing multiple code blocks', async () => { + const inputFragments = [ + 'Text before code.\n```js\ncode1\n```\nText between code.\n```js\ncode2\n```\nText after code.', + ]; + + const inputIterable = asyncIterableFromArray(inputFragments); + const identifier = { start: '```js', end: '```' }; + + await processStreamWithIdentifiers({ + processStreamFragment, + onStreamIdentifier, + inputIterable, + identifier, + }); + + expect(fragmentsProcessed.join('')).to.deep.equal(inputFragments.join('')); + expect(identifiersStreamed).to.deep.equal(['\ncode1\n', '\ncode2\n']); + }); +}); diff --git a/src/test/suite/telemetry/telemetryService.test.ts b/src/test/suite/telemetry/telemetryService.test.ts index 42f1c1659..f6e78e374 100644 --- a/src/test/suite/telemetry/telemetryService.test.ts +++ b/src/test/suite/telemetry/telemetryService.test.ts @@ -14,6 +14,7 @@ import { DocumentSource } from '../../../documentSource'; import { mdbTestExtension } from '../stubbableMdbExtension'; import { DatabaseTreeItem, DocumentTreeItem } from '../../../explorer'; import { DataServiceStub } from '../stubs'; +import { chatResultFeedbackKindToTelemetryValue } from '../../../telemetry/telemetryService'; // eslint-disable-next-line @typescript-eslint/no-var-requires const { version } = require('../../../../package.json'); @@ -234,7 +235,9 @@ suite('Telemetry Controller Test Suite', () => { test('track playground code executed event', async () => { const testPlaygroundController = mdbTestExtension.testExtensionController._playgroundController; - await testPlaygroundController._evaluate('show dbs'); + await testPlaygroundController._evaluate({ + codeToEvaluate: 'show dbs', + }); sandbox.assert.calledWith( fakeSegmentAnalyticsTrack, sinon.match({ @@ -750,4 +753,22 @@ suite('Telemetry Controller Test Suite', () => { }) ); }); + + function enumKeys< + TEnum extends object, + TKey extends keyof TEnum = keyof TEnum + >(obj: TEnum): TKey[] { + return Object.keys(obj).filter((k) => Number.isNaN(k)) as TKey[]; + } + + test('ChatResultFeedbackKind to TelemetryFeedbackKind maps all values', () => { + for (const kind of enumKeys(vscode.ChatResultFeedbackKind)) { + expect( + chatResultFeedbackKindToTelemetryValue( + vscode.ChatResultFeedbackKind[kind] + ), + `Expect ${kind} to produce a concrete telemetry value` + ).to.not.be.undefined; + } + }); });