Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

community[major]: Merge community #3610

Merged
merged 18 commits into from
Dec 11, 2023
Merged

community[major]: Merge community #3610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
1fe6934
langchain[major]: LangChain community (#3581)
jacoblee93 Dec 8, 2023
fb58304
Merge branch 'main' of https://github.com/hwchase17/langchainjs into rc
jacoblee93 Dec 8, 2023
3aa51e5
Move memory vector store back into langchain
jacoblee93 Dec 8, 2023
155086d
Move memory
jacoblee93 Dec 8, 2023
9575611
Move toolkits
jacoblee93 Dec 8, 2023
3860070
Move more tools and toolkits
jacoblee93 Dec 8, 2023
242cab1
Update yarn lock
jacoblee93 Dec 8, 2023
b961daf
Fix lint
jacoblee93 Dec 8, 2023
4896c94
Move test
jacoblee93 Dec 8, 2023
44be4bc
Brace/add missing neo4j test (#3597)
bracesproul Dec 8, 2023
938b8e3
community[minor]: Fix CI (#3601)
jacoblee93 Dec 9, 2023
6e5e06d
Brace/cleanup deps (#3600)
bracesproul Dec 9, 2023
0e91a69
Add skeleton of import tests
jacoblee93 Dec 9, 2023
9c27962
Merge branch 'rc' of https://github.com/hwchase17/langchainjs into rc
jacoblee93 Dec 9, 2023
31a1bce
comminity[major]: Cleaned deps (#3602)
bracesproul Dec 9, 2023
bca427d
Bump version
jacoblee93 Dec 9, 2023
eba9ccd
Remove unused types
jacoblee93 Dec 10, 2023
23641c6
Use different import maps for core vs main langchain (#3617)
jacoblee93 Dec 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
25 changes: 25 additions & 0 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ version: "3"
services:
test-exports-esbuild:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -11,11 +14,15 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
test-exports-esm:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -25,11 +32,15 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
test-exports-cjs:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -39,11 +50,15 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
test-exports-cf:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -53,11 +68,15 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
test-exports-vercel:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -67,11 +86,15 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
test-exports-vite:
image: node:18
environment:
PUPPETEER_SKIP_DOWNLOAD: "true"
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "true"
working_dir: /app
volumes:
- ./yarn.lock:/root/yarn.lock
Expand All @@ -81,6 +104,7 @@ services:
- ./environment_tests/scripts:/scripts
- ./langchain:/langchain
- ./langchain-core:/langchain-core
- ./libs/langchain-community:/langchain-community
- ./libs/langchain-anthropic:/langchain-anthropic
- ./libs/langchain-openai:/langchain-openai
command: bash /scripts/docker-ci-entrypoint.sh
Expand All @@ -92,6 +116,7 @@ services:
# - ./environment_tests/scripts:/scripts
# - ./langchain:/langchain-workspace
# - ./langchain-core:/langchain-core
# - ./libs/langchain-community:/langchain-community-workspace
# - ./libs/langchain-anthropic:/langchain-anthropic-workspace
# command: bash /scripts/docker-bun-ci-entrypoint.sh
success:
Expand Down
20 changes: 12 additions & 8 deletions environment_tests/scripts/docker-ci-entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,23 @@ shopt -s extglob
# avoid copying build artifacts from the host
cp -r ../package/!(node_modules|dist|dist-cjs|dist-esm|build|.next|.turbo) .

mkdir -p ./libs/langchain-core/
mkdir -p ./libs/langchain-openai/
mkdir -p ./libs/langchain-anthropic/
mkdir -p ./libs/langchain-community/
mkdir -p ./libs/langchain/

cp -r ../langchain-core ./libs/
cp -r ../langchain-openai ./libs/
cp -r ../langchain-anthropic ./libs/
cp -r ../langchain-community ./libs/
cp -r ../langchain ./libs/

# copy cache
mkdir -p ./.yarn
cp -r ../root/.yarn/!(berry|cache) ./.yarn
cp ../root/yarn.lock ../root/.yarnrc.yml .

# Replace the workspace dependency with the local copy, and install all others
# Avoid calling "yarn add ../langchain" as yarn berry does seem to hang for ~30s
# before installation actually occurs
sed -i 's/"@langchain\/core": "workspace:\*"/"@langchain\/core": "..\/langchain-core"/g' package.json
sed -i 's/"@langchain\/anthropic": "workspace:\*"/"@langchain\/anthropic": "..\/langchain-anthropic"/g' package.json
sed -i 's/"@langchain\/openai": "workspace:\*"/"@langchain\/openai": "..\/langchain-openai"/g' package.json
sed -i 's/"langchain": "workspace:\*"/"langchain": "..\/langchain"/g' package.json

yarn install --no-immutable

# Check the build command completes successfully
Expand Down
1 change: 1 addition & 0 deletions environment_tests/test-exports-bun/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"license": "MIT",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! I noticed the addition of a new dependency, "@langchain/community", in the package.json file. This seems to be a new peer/dev/hard dependency, and I'm flagging this for the maintainers to review. Great work on the PR!

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"d3-dsv": "2",
Expand Down
4 changes: 4 additions & 0 deletions environment_tests/test-exports-cf/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "test-exports-cf",
"version": "0.0.0",
"workspaces": [
"libs/*"
],
"devDependencies": {
"@cloudflare/workers-types": "^4.20230321.0",
"typescript": "^5.0.3",
Expand All @@ -9,6 +12,7 @@
},
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! 👋 I noticed that this PR introduces a new dependency, "@langchain/community", with a version specified as "workspace:*". This comment is to flag the dependency change for maintainers to review, especially regarding its type (peer/dev/hard). Keep up the great work!

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"langchain": "workspace:*"
Expand Down
4 changes: 4 additions & 0 deletions environment_tests/test-exports-cjs/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "test-exports-cjs",
"version": "0.0.0",
"workspaces": [
"libs/*"
],
"private": true,
"description": "CJS Tests for the things exported by the langchain package",
"main": "./index.mjs",
Expand All @@ -19,6 +22,7 @@
"license": "MIT",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! 👋 I noticed that this PR adds a new hard dependency "@langchain/community" to the project's package.json. Just flagging this for the maintainers to review. Great work overall! 🚀

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"d3-dsv": "2",
Expand Down
4 changes: 4 additions & 0 deletions environment_tests/test-exports-esbuild/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "test-exports-esbuild",
"version": "0.0.0",
"workspaces": [
"libs/*"
],
"private": true,
"description": "Tests for the things exported by the langchain package",
"main": "./index.mjs",
Expand All @@ -17,6 +20,7 @@
"license": "MIT",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! 👋 I noticed that this PR adds a new dependency to the project, "@langchain/community". It's important for maintainers to review this change to ensure it aligns with the project's dependency requirements. Thank you for flagging this for review! 🚀

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"d3-dsv": "2",
Expand Down
4 changes: 4 additions & 0 deletions environment_tests/test-exports-esm/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "test-exports-esm",
"version": "0.0.0",
"workspaces": [
"libs/*"
],
"private": true,
"description": "Tests for the things exported by the langchain package",
"main": "./index.mjs",
Expand All @@ -20,6 +23,7 @@
"license": "MIT",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! I noticed that the addition of "@langchain/community" as a dependency in the package.json file might impact the project's peer dependencies. I've flagged this for your review as it's an important change to the project's dependencies. Keep up the great work!

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"d3-dsv": "2",
Expand Down
4 changes: 4 additions & 0 deletions environment_tests/test-exports-vercel/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "test-exports-vercel",
"version": "0.1.0",
"workspaces": [
"libs/*"
],
"private": true,
"scripts": {
"dev": "next dev",
Expand All @@ -10,6 +13,7 @@
},
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! 👋 Just wanted to flag for the maintainers that a new dev dependency, "@langchain/community", has been added in this PR. It's always good to review dependency changes to ensure everything is in order. Thanks!

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"@types/node": "18.15.11",
Expand Down
5 changes: 4 additions & 1 deletion environment_tests/test-exports-vercel/tsconfig.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,8 @@
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
"exclude": ["node_modules"]
"exclude": [
"node_modules",
"libs"
]
}
6 changes: 5 additions & 1 deletion environment_tests/test-exports-vite/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
{
"name": "test-exports-vite",
"private": true,
"version": "0.0.0",
"workspaces": [
"libs/*"
],
"private": true,
"type": "module",
"scripts": {
"dev": "vite",
Expand All @@ -11,6 +14,7 @@
},
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! I noticed that the addition of "@langchain/community" as a dependency in the package.json file might impact the project's dependencies. I've flagged this for your review to ensure it aligns with the intended dependency type (peer/dev/hard). Keep up the great work!

"dependencies": {
"@langchain/anthropic": "workspace:*",
"@langchain/community": "workspace:*",
"@langchain/core": "workspace:*",
"@langchain/openai": "workspace:*",
"langchain": "workspace:*"
Expand Down
2 changes: 1 addition & 1 deletion examples/src/guides/expression_language/runnable_history.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ChatOpenAI } from "langchain/chat_models/openai";
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
import { ChatMessageHistory } from "langchain/memory";
import { ChatPromptTemplate, MessagesPlaceholder } from "langchain/prompts";
import {
RunnableConfig,
Expand Down
2 changes: 1 addition & 1 deletion examples/src/guides/expression_language/runnable_history_constructor_config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ChatOpenAI } from "langchain/chat_models/openai";
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
import { ChatMessageHistory } from "langchain/memory";
import { ChatPromptTemplate, MessagesPlaceholder } from "langchain/prompts";
import {
RunnableConfig,
Expand Down
3 changes: 3 additions & 0 deletions langchain-core/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ utils/json_patch.d.ts
utils/json_schema.cjs
utils/json_schema.js
utils/json_schema.d.ts
utils/math.cjs
utils/math.js
utils/math.d.ts
utils/stream.cjs
utils/stream.js
utils/stream.d.ts
Expand Down
12 changes: 11 additions & 1 deletion langchain-core/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@langchain/core",
"version": "0.0.10",
"version": "0.0.11-rc.1",
"description": "Core LangChain.js abstractions and schemas",
"type": "module",
"engines": {
Expand Down Expand Up @@ -40,6 +40,7 @@
"decamelize": "1.2.0",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey there! I noticed the addition of "ml-distance" as a dependency in the package.json file. This seems to be a peer/dev/hard dependency change, and I'm flagging this for the maintainers to review. Keep up the great work!

"js-tiktoken": "^1.0.8",
"langsmith": "~0.0.48",
"ml-distance": "^4.0.0",
"p-queue": "^6.6.2",
"p-retry": "4",
"uuid": "^9.0.0",
Expand All @@ -59,6 +60,7 @@
"eslint-plugin-prettier": "^4.2.1",
"jest": "^29.5.0",
"jest-environment-node": "^29.6.4",
"ml-matrix": "^6.10.4",
"prettier": "^2.8.3",
"release-it": "^15.10.1",
"rimraf": "^5.0.1",
Expand Down Expand Up @@ -263,6 +265,11 @@
"import": "./utils/json_schema.js",
"require": "./utils/json_schema.cjs"
},
"./utils/math": {
"types": "./utils/math.d.ts",
"import": "./utils/math.js",
"require": "./utils/math.cjs"
},
"./utils/stream": {
"types": "./utils/stream.d.ts",
"import": "./utils/stream.js",
Expand Down Expand Up @@ -400,6 +407,9 @@
"utils/json_schema.cjs",
"utils/json_schema.js",
"utils/json_schema.d.ts",
"utils/math.cjs",
"utils/math.js",
"utils/math.d.ts",
"utils/stream.cjs",
"utils/stream.js",
"utils/stream.d.ts",
Expand Down
1 change: 1 addition & 0 deletions langchain-core/scripts/create-entrypoints.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const entrypoints = {
"utils/hash": "utils/hash",
"utils/json_patch": "utils/json_patch",
"utils/json_schema": "utils/json_schema",
"utils/math": "utils/math",
"utils/stream": "utils/stream",
"utils/testing": "utils/testing/index",
"utils/tiktoken": "utils/tiktoken",
Expand Down
17 changes: 17 additions & 0 deletions langchain-core/src/documents/transformers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,20 @@ export abstract class BaseDocumentTransformer<
return this.transformDocuments(input);
}
}

/**
* Class for document transformers that return exactly one transformed document
* for each input document.
*/
export abstract class MappingDocumentTransformer extends BaseDocumentTransformer {
async transformDocuments(documents: Document[]): Promise<Document[]> {
const newDocuments = [];
for (const document of documents) {
const transformedDocument = await this._transformDocument(document);
newDocuments.push(transformedDocument);
}
return newDocuments;
}

abstract _transformDocument(document: Document): Promise<Document>;
}
1 change: 1 addition & 0 deletions langchain-core/src/load/import_map.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export * as utils__env from "../utils/env.js";
export * as utils__hash from "../utils/hash.js";
export * as utils__json_patch from "../utils/json_patch.js";
export * as utils__json_schema from "../utils/json_schema.js";
export * as utils__math from "../utils/math.js";
export * as utils__stream from "../utils/stream.js";
export * as utils__testing from "../utils/testing/index.js";
export * as utils__tiktoken from "../utils/tiktoken.js";
Expand Down
Loading