all[chore]: Use turbo repo to build api refs, docs and more (#3511)

* api_refs[chore]: use turbo repo to build api refs and other dependencies * update core docs to also use turbo * cr * use turbo in ci * cr * chore: lint files * cr
langchain-ai · Dec 4, 2023 · f289f3d · f289f3d · vercel · Dec 4, 2023
1 parent db5f920
commit f289f3d
Show file tree

Hide file tree

Showing 12 changed files with 146 additions and 113 deletions.
diff --git a/.github/workflows/test-exports.yml b/.github/workflows/test-exports.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -54,7 +54,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -74,7 +74,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -94,7 +94,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -114,7 +114,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -134,7 +134,7 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
       - name: Build
-        run: yarn run build:deps && yarn workspace langchain build
+        run: yarn run build:deps && yarn build --filter=langchain
         shell: bash
         env:
           SKIP_API_DOCS: true
@@ -154,7 +154,7 @@ jobs:
   #     - name: Install dependencies
   #       run: yarn install --immutable
   #     - name: Build
-  #       run: yarn run build:deps && yarn workspace langchain build
+  #       run: yarn run build:deps && yarn build --filter=langchain
   #       shell: bash
   #       env:
   #         SKIP_API_DOCS: true

diff --git a/docs/api_refs/package.json b/docs/api_refs/package.json
@@ -1,12 +1,13 @@
 {
-  "name": "api_refs",
+  "name": "@langchain/api_refs",
   "version": "0.1.0",
   "private": true,
   "scripts": {
     "dev": "next dev -p 3001",
     "typedoc": "npx typedoc --options typedoc.json",
     "build:scripts": "node ./scripts/generate-api-refs.js && node ./scripts/update-typedoc-css.js",
-    "build": "yarn run build:deps && yarn workspace langchain build && yarn build:scripts && next build",
+    "build": "yarn turbo run build:next",
+    "build:next": "next build",
     "start": "yarn build && next start -p 3001",
     "lint": "next lint"
   },
@@ -25,6 +26,7 @@
     "postcss": "^8",
     "tailwindcss": "^3.3.0",
     "ts-morph": "^20.0.0",
+    "turbo": "latest",
     "typescript": "^5"
   }
 }
diff --git a/docs/api_refs/turbo.json b/docs/api_refs/turbo.json
@@ -0,0 +1,18 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "extends": ["//"],
+  "pipeline": {
+    "build:scripts": {
+      "outputs": ["public/**"],
+      "dependsOn": [
+        "@langchain/openai#build",
+        "@langchain/anthropic#build",
+        "langchain#build"
+      ]
+    },
+    "build:next": {
+      "outputs": [".next/**", ".vercel/**"],
+      "dependsOn": ["build:scripts"]
+    }
+  }
+}
diff --git a/docs/core_docs/package.json b/docs/core_docs/package.json
@@ -1,11 +1,13 @@
 {
-  "name": "core_docs",
+  "name": "@langchain/core_docs",
   "version": "0.0.0",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
     "start": "yarn build:typedoc && rimraf ./docs/api && NODE_OPTIONS=--max-old-space-size=7168 docusaurus start",
-    "build": "yarn build:typedoc && rimraf ./build && NODE_OPTIONS=--max-old-space-size=7168 DOCUSAURUS_SSR_CONCURRENCY=4 docusaurus build",
+    "rimraf:build": "rimraf ./build",
+    "build:docusaurus": "NODE_OPTIONS=--max-old-space-size=7168 DOCUSAURUS_SSR_CONCURRENCY=4 docusaurus build",
+    "build": "yarn turbo run build:docusaurus",
     "build:typedoc": "cd ../api_refs && yarn build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
@@ -47,6 +49,7 @@
     "prettier": "^2.7.1",
     "rimraf": "^5.0.1",
     "swc-loader": "^0.2.3",
+    "turbo": "latest",
     "typedoc": "^0.24.4",
     "typedoc-plugin-markdown": "next"
   },

diff --git a/docs/core_docs/turbo.json b/docs/core_docs/turbo.json
@@ -0,0 +1,13 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "extends": ["//"],
+  "pipeline": {
+    "rimraf:build": {
+      "dependsOn": ["@langchain/api_refs#build"]
+    },
+    "build:docusaurus": {
+      "outputs": [".docusaurus/**", "build/**"],
+      "dependsOn": ["rimraf:build"]
+    }
+  }
+}
diff --git a/langchain-core/src/callbacks/manager.ts b/langchain-core/src/callbacks/manager.ts
@@ -1,7 +1,7 @@
 import { v4 as uuidv4 } from "uuid";
-import { AgentAction, AgentFinish } from "../agents.js";
+import type { AgentAction, AgentFinish } from "../agents.js";
 import type { ChainValues } from "../utils/types.js";
-import { LLMResult } from "../outputs.js";
+import type { LLMResult } from "../outputs.js";
 import {
   BaseCallbackHandler,
   CallbackHandlerMethods,

diff --git a/langchain-core/src/language_models/chat_models.ts b/langchain-core/src/language_models/chat_models.ts
@@ -2,17 +2,17 @@ import {
   AIMessage,
   BaseMessage,
   BaseMessageChunk,
-  BaseMessageLike,
+  type BaseMessageLike,
   HumanMessage,
   coerceMessageLikeToMessage,
 } from "../messages/index.js";
 import { BasePromptValue } from "../prompt_values.js";
 import {
-  LLMResult,
+  type LLMResult,
   RUN_KEY,
-  ChatGeneration,
+  type ChatGeneration,
   ChatGenerationChunk,
-  ChatResult,
+  type ChatResult,
 } from "../outputs.js";
 import {
   BaseLanguageModel,

diff --git a/langchain-core/src/language_models/llms.ts b/langchain-core/src/language_models/llms.ts
@@ -1,19 +1,24 @@
 import { AIMessage, BaseMessage, getBufferString } from "../messages/index.js";
 import { BasePromptValue } from "../prompt_values.js";
-import { LLMResult, RUN_KEY, Generation, GenerationChunk } from "../outputs.js";
 import {
-  BaseCallbackConfig,
+  type LLMResult,
+  RUN_KEY,
+  type Generation,
+  GenerationChunk,
+} from "../outputs.js";
+import {
+  type BaseCallbackConfig,
   CallbackManager,
   CallbackManagerForLLMRun,
-  Callbacks,
+  type Callbacks,
 } from "../callbacks/manager.js";
 import {
   BaseLanguageModel,
   type BaseLanguageModelCallOptions,
   type BaseLanguageModelInput,
   type BaseLanguageModelParams,
 } from "./base.js";
-import { RunnableConfig } from "../runnables/config.js";
+import type { RunnableConfig } from "../runnables/config.js";
 
 export type SerializedLLM = {
   _model: string;

diff --git a/langchain-core/src/messages/tests/base_message.test.ts b/langchain-core/src/messages/tests/base_message.test.ts
@@ -1,6 +1,6 @@
 import { test } from "@jest/globals";
 import { ChatPromptTemplate } from "../../prompts/chat.js";
-import { HumanMessage } from "../../messages/index.js";
+import { HumanMessage } from "../index.js";
 
 test("Test ChatPromptTemplate can format OpenAI content image messages", async () => {
   const message = new HumanMessage({

diff --git a/package.json b/package.json
@@ -19,12 +19,12 @@
   "packageManager": "[email protected]",
   "scripts": {
     "build": "turbo run build --filter=\"!test-exports-*\" --concurrency 1",
-    "build:deps": "yarn workspace @langchain/core build && yarn workspace @langchain/anthropic build && yarn workspace @langchain/openai build",
+    "build:deps": "yarn build --filter=@langchain/openai --filter=@langchain/anthropic --filter=@langchain/core",
     "format": "turbo run format",
     "format:check": "turbo run format:check",
     "lint": "turbo run lint --concurrency 1",
     "lint:fix": "yarn lint -- --fix",
-    "test": "yarn test:unit && yarn workspace @langchain/core build && yarn workspace langchain build && yarn test:exports:docker",
+    "test": "yarn test:unit && yarn build --filter=langchain && yarn test:exports:docker",
     "test:unit": "turbo run test --filter @langchain/core --filter langchain",
     "test:int": "yarn run test:int:deps && turbo run test:integration ; yarn run test:int:deps:down",
     "test:int:deps": "docker compose -f test-int-deps-docker-compose.yml up -d",
@@ -34,8 +34,8 @@
     "publish:core": "bash langchain/scripts/release-branch.sh && turbo run --filter @langchain/core build lint test --concurrency 1 && yarn run test:exports:docker && yarn workspace @langchain/core run release && echo '🔗 Open https://github.com/langchain-ai/langchainjs/compare/release?expand=1 and merge the release PR'",
     "example": "yarn workspace examples start",
     "precommit": "turbo run precommit",
-    "docs": "yarn workspace core_docs start",
-    "docs:api_refs": "yarn workspace api_refs start"
+    "docs": "yarn workspace @langchain/core_docs start",
+    "docs:api_refs": "yarn workspace @langchain/api_refs start"
   },
   "author": "LangChain",
   "license": "MIT",

diff --git a/turbo.json b/turbo.json
@@ -1,38 +1,32 @@
 {
   "$schema": "https://turbo.build/schema.json",
-  "globalDependencies": [
-    "**/.env"
-  ],
+  "globalDependencies": ["**/.env"],
   "pipeline": {
-    "@langchain/core#build": {},
-    "libs/langchain-anthropic#build": {
-      "dependsOn": [
-        "@langchain/core#build"
+    "@langchain/core#build": {
+      "outputs": [
+        "langchain-core/dist/**",
+        "langchain-core/dist-cjs/**",
+        "langchain-core/*.js",
+        "langchain-core/*.cjs",
+        "langchain-core/*.d.ts"
+      ],
+      "inputs": [
+        "langchain-core/src/**",
+        "langchain-core/scripts/**",
+        "langchain-core/package.json",
+        "langchain-core/tsconfig.json"
       ]
     },
+    "libs/langchain-anthropic#build": {
+      "dependsOn": ["@langchain/core#build"]
+    },
     "libs/langchain-openai#build": {
-      "dependsOn": [
-        "@langchain/core#build"
-      ]
+      "dependsOn": ["@langchain/core#build"]
     },
     "build": {
-      "dependsOn": [
-        "@langchain/core#build",
-        "^build"
-      ],
-      "outputs": [
-        "dist/**",
-        "dist-cjs/**",
-        "*.js",
-        "*.cjs",
-        "*.d.ts"
-      ],
-      "inputs": [
-        "src/**",
-        "scripts/**",
-        "package.json",
-        "tsconfig.json"
-      ]
+      "dependsOn": ["@langchain/core#build", "^build"],
+      "outputs": ["dist/**", "dist-cjs/**", "*.js", "*.cjs", "*.d.ts"],
+      "inputs": ["src/**", "scripts/**", "package.json", "tsconfig.json"]
     },
     "lint": {
       "outputs": []
@@ -45,15 +39,11 @@
     },
     "test": {
       "outputs": [],
-      "dependsOn": [
-        "^build"
-      ]
+      "dependsOn": ["^build"]
     },
     "test:integration": {
       "outputs": [],
-      "dependsOn": [
-        "^build"
-      ]
+      "dependsOn": ["^build"]
     },
     "precommit": {},
     "start": {