out of memory if too many conditional exports #423
Comments
|
Do you have a reproduction link? did'nt see bunchee is setup in that repo |
im still working on this |
|
but you can overwrite the package.json and see it |
|
Possible you bundled sth re not in the dependencies or peer dependencies, it happened before when some users were trying to bundle webpack but actually webpack should be external |
|
Thanks I can repro it now, will investigate which part is having mem leak 👁️🗨️ One possible is probably due to rollup production optimization that causes a lot of memory, or it's using too much mem in ts generation plugin |
huozhi
added
area:bug
|
Looks like if there's no types generation they memory allocation will be fine without specifying extra memory, sth wrong with the ts generation. |
|
did some debugging using the reproducible example and for each new export the time of the other export bundle increases significantly. I tried to run it synchronously then decreased it from 30s to 1.50s per exported module (max 4.50s big ones), still playing with this
|
|
This is still happening on our case especially when increasing the exports, sadly I'm considering to switch to other bundler |
|
I'm rewriting the exports parser and in the shorter future we gonna generate less type files for your case which could help a bit on the memory. I can't commit a time that can fix this soon since the resource from me on this is pretty limited atm. |
|
So the generating more types file is not the main problem. Seems ts plugin will consume too much memory anyway. I might need to move those work to a separate worker |
|
Tested with the repro, it's resolved with 5.0.0-beta.2 |
|
@himself65 [email protected] improved more about the types generation perf for multi exports! 🙏 |
{ "name": "llamaindex", "version": "0.0.46", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.9.1", "@aws-crypto/sha256-browser": "^5.2.0", "@datastax/astra-db-ts": "^0.1.2", "@mistralai/mistralai": "^0.0.7", "@notionhq/client": "^2.2.14", "@pinecone-database/pinecone": "^1.1.2", "@xenova/transformers": "^2.10.0", "assemblyai": "^4.0.0", "chromadb": "^1.7.3", "file-type": "^18.7.0", "js-tiktoken": "^1.0.8", "lodash": "^4.17.21", "mammoth": "^1.6.0", "md-utils-ts": "^2.0.0", "mongodb": "^6.3.0", "notion-md-crawler": "^0.0.2", "openai": "^4.20.1", "papaparse": "^5.4.1", "pdfjs-dist": "4.0.269", "pg": "^8.11.3", "pgvector": "^0.1.5", "portkey-ai": "^0.1.16", "rake-modified": "^1.0.8", "replicate": "^0.21.1", "string-strip-html": "^13.4.3", "wink-nlp": "^1.14.3" }, "devDependencies": { "@types/jest": "^29.5.11", "@types/lodash": "^4.14.202", "@types/node": "^18.19.6", "@types/papaparse": "^5.3.14", "@types/pg": "^8.10.9", "bunchee": "^4.4.0", "node-stdlib-browser": "^1.2.0", "typescript": "^5.3.3" }, "engines": { "node": ">=18.0.0" }, "types": "./dist/index.d.ts", "main": "./dist/index.js", "exports": { ".": { "types": "./dist/index.d.mts", "edge-light": "./dist/index.edge-light.mjs", "import": "./dist/index.mjs", "require": "./dist/index.js" }, "./storage/FileSystem": { "types": "./dist/storage/FileSystem.d.mts", "edge-light": "./dist/storage/FileSystem.edge-light.mjs", "import": "./dist/storage/FileSystem.mjs", "require": "./dist/storage/FileSystem.js" }, "./ChatEngine": { "types": "./dist/ChatEngine.d.mts", "edge-light": "./dist/ChatEngine.edge-light.mjs", "import": "./dist/ChatEngine.mjs", "require": "./dist/ChatEngine.js" }, "./ChatHistory": { "types": "./dist/ChatHistory.d.mts", "edge-light": "./dist/ChatHistory.edge-light.mjs", "import": "./dist/ChatHistory.mjs", "require": "./dist/ChatHistory.js" }, "./constants": { "types": "./dist/constants.d.mts", "edge-light": "./dist/constants.edge-light.mjs", "import": "./dist/constants.mjs", "require": "./dist/constants.js" }, "./GlobalsHelper": { "types": "./dist/GlobalsHelper.d.mts", "edge-light": "./dist/GlobalsHelper.edge-light.mjs", "import": "./dist/GlobalsHelper.mjs", "require": "./dist/GlobalsHelper.js" }, "./Node": { "types": "./dist/Node.d.mts", "edge-light": "./dist/Node.edge-light.mjs", "import": "./dist/Node.mjs", "require": "./dist/Node.js" }, "./OutputParser": { "types": "./dist/OutputParser.d.mts", "edge-light": "./dist/OutputParser.edge-light.mjs", "import": "./dist/OutputParser.mjs", "require": "./dist/OutputParser.js" }, "./Prompt": { "types": "./dist/Prompt.d.mts", "edge-light": "./dist/Prompt.edge-light.mjs", "import": "./dist/Prompt.mjs", "require": "./dist/Prompt.js" }, "./PromptHelper": { "types": "./dist/PromptHelper.d.mts", "edge-light": "./dist/PromptHelper.edge-light.mjs", "import": "./dist/PromptHelper.mjs", "require": "./dist/PromptHelper.js" }, "./QueryEngine": { "types": "./dist/QueryEngine.d.mts", "edge-light": "./dist/QueryEngine.edge-light.mjs", "import": "./dist/QueryEngine.mjs", "require": "./dist/QueryEngine.js" }, "./QuestionGenerator": { "types": "./dist/QuestionGenerator.d.mts", "edge-light": "./dist/QuestionGenerator.edge-light.mjs", "import": "./dist/QuestionGenerator.mjs", "require": "./dist/QuestionGenerator.js" }, "./Response": { "types": "./dist/Response.d.mts", "edge-light": "./dist/Response.edge-light.mjs", "import": "./dist/Response.mjs", "require": "./dist/Response.js" }, "./Retriever": { "types": "./dist/Retriever.d.mts", "edge-light": "./dist/Retriever.edge-light.mjs", "import": "./dist/Retriever.mjs", "require": "./dist/Retriever.js" }, "./ServiceContext": { "types": "./dist/ServiceContext.d.mts", "edge-light": "./dist/ServiceContext.edge-light.mjs", "import": "./dist/ServiceContext.mjs", "require": "./dist/ServiceContext.js" }, "./TextSplitter": { "types": "./dist/TextSplitter.d.mts", "edge-light": "./dist/TextSplitter.edge-light.mjs", "import": "./dist/TextSplitter.mjs", "require": "./dist/TextSplitter.js" }, "./Tool": { "types": "./dist/Tool.d.mts", "edge-light": "./dist/Tool.edge-light.mjs", "import": "./dist/Tool.mjs", "require": "./dist/Tool.js" }, "./readers/AssemblyAI": { "types": "./dist/readers/AssemblyAI.d.mts", "edge-light": "./dist/readers/AssemblyAI.edge-light.mjs", "import": "./dist/readers/AssemblyAI.mjs", "require": "./dist/readers/AssemblyAI.js" }, "./readers/base": { "types": "./dist/readers/base.d.mts", "edge-light": "./dist/readers/base.edge-light.mjs", "import": "./dist/readers/base.mjs", "require": "./dist/readers/base.js" }, "./readers/CSVReader": { "types": "./dist/readers/CSVReader.d.mts", "edge-light": "./dist/readers/CSVReader.edge-light.mjs", "import": "./dist/readers/CSVReader.mjs", "require": "./dist/readers/CSVReader.js" }, "./readers/DocxReader": { "types": "./dist/readers/DocxReader.d.mts", "edge-light": "./dist/readers/DocxReader.edge-light.mjs", "import": "./dist/readers/DocxReader.mjs", "require": "./dist/readers/DocxReader.js" }, "./readers/HTMLReader": { "types": "./dist/readers/HTMLReader.d.mts", "edge-light": "./dist/readers/HTMLReader.edge-light.mjs", "import": "./dist/readers/HTMLReader.mjs", "require": "./dist/readers/HTMLReader.js" }, "./readers/ImageReader": { "types": "./dist/readers/ImageReader.d.mts", "edge-light": "./dist/readers/ImageReader.edge-light.mjs", "import": "./dist/readers/ImageReader.mjs", "require": "./dist/readers/ImageReader.js" }, "./readers/MarkdownReader": { "types": "./dist/readers/MarkdownReader.d.mts", "edge-light": "./dist/readers/MarkdownReader.edge-light.mjs", "import": "./dist/readers/MarkdownReader.mjs", "require": "./dist/readers/MarkdownReader.js" }, "./readers/NotionReader": { "types": "./dist/readers/NotionReader.d.mts", "edge-light": "./dist/readers/NotionReader.edge-light.mjs", "import": "./dist/readers/NotionReader.mjs", "require": "./dist/readers/NotionReader.js" }, "./readers/PDFReader": { "types": "./dist/readers/PDFReader.d.mts", "edge-light": "./dist/readers/PDFReader.edge-light.mjs", "import": "./dist/readers/PDFReader.mjs", "require": "./dist/readers/PDFReader.js" }, "./readers/SimpleDirectoryReader": { "types": "./dist/readers/SimpleDirectoryReader.d.mts", "edge-light": "./dist/readers/SimpleDirectoryReader.edge-light.mjs", "import": "./dist/readers/SimpleDirectoryReader.mjs", "require": "./dist/readers/SimpleDirectoryReader.js" }, "./readers/SimpleMongoReader": { "types": "./dist/readers/SimpleMongoReader.d.mts", "edge-light": "./dist/readers/SimpleMongoReader.edge-light.mjs", "import": "./dist/readers/SimpleMongoReader.mjs", "require": "./dist/readers/SimpleMongoReader.js" }, "./environments": { "types": "./dist/environments.d.mts", "edge-light": "./dist/environments.edge-light.mjs", "import": "./dist/environments.mjs", "require": "./dist/environments.js" }, "./examples/*": "./examples/*" }, "files": [ "dist", "examples", "src", "types", "CHANGELOG.md" ], "repository": { "type": "git", "url": "https://github.com/run-llama/LlamaIndexTS.git", "directory": "packages/core" }, "scripts": { "lint": "eslint .", "test": "jest", "build": "bunchee", "dev": "bunchee -w" } }The text was updated successfully, but these errors were encountered: