Merge pull request #40 from jezekra1/feat-add-ibm-vllm-adapter

feat(ibm-vllm): add llm adapter

.gitignore

@@ -1,3 +1,10 @@
+scripts/ibm_vllm_generate_protos/dist
+scripts/ibm_vllm_generate_protos/dts
+scripts/ibm_vllm_generate_protos/types
+
+infra/bee-code-interpreter/*
+!infra/bee-code-interpreter/bee-code-interpreter.yaml
+
 ### Node template
 # Logs
 logs

eslint.config.js

@@ -8,11 +8,15 @@ import markdown from "@eslint/markdown";
 
 export default tseslint.config(
   {
-    ignores: ["node_modules/**", "dist/**"],
+    ignores: ["node_modules/**", "dist/**", "scripts/ibm_vllm_generate_protos/**"],
   },
   eslint.configs.recommended,
   ...tseslint.configs.strict,
   ...tseslint.configs.stylistic,
+  {
+    files: ["src/adapters/ibm-vllm/types.ts"],
+    rules: { "@typescript-eslint/unified-signatures": "off" },
+  },
   {
     files: ["**/*.md/**"],
     languageOptions: {

examples/llms/providers/ibm-vllm.ts

@@ -0,0 +1,36 @@
+import "dotenv/config.js";
+import { IBMvLLM } from "bee-agent-framework/adapters/ibm-vllm/llm";
+import { IBMVllmChatLLM } from "bee-agent-framework/adapters/ibm-vllm/chat";
+import { BaseMessage } from "bee-agent-framework/llms/primitives/message";
+import { Client } from "bee-agent-framework/adapters/ibm-vllm/client";
+
+const client = new Client();
+{
+  console.info("===RAW===");
+  const llm = new IBMvLLM({
+    client,
+    modelId: "meta-llama/llama-3-1-70b-instruct",
+  });
+
+  console.info("Meta", await llm.meta());
+
+  const response = await llm.generate("Hello world!", {
+    stream: false,
+  });
+  console.info(response.text);
+}
+
+{
+  console.info("===CHAT===");
+  const llm = IBMVllmChatLLM.fromPreset("meta-llama/llama-3-1-70b-instruct", { client });
+
+  console.info("Meta", await llm.meta());
+
+  const response = await llm.generate([
+    BaseMessage.of({
+      role: "user",
+      text: "Hello world!",
+    }),
+  ]);
+  console.info(response.messages);
+}

package.json

@@ -78,6 +78,7 @@
     "prepare": "husky",
     "copyright": "./scripts/copyright.sh",
     "release": "release-it",
+    "ibm-vllm:generate-types": "./scripts/ibm_vllm_generate_protos/ibm_vllm_generate_protos.sh",
     "_ensure_env": "cp -n .env.template .env || true",
     "_docker": "yarn _ensure_env && bash -c 'source ./.env && docker_cmd=$(which docker >/dev/null 2>&1 && printf docker || printf podman) && $docker_cmd \"$@\"' sh"
   },
@@ -115,6 +116,8 @@
   },
   "peerDependencies": {
     "@googleapis/customsearch": "^3.2.0",
+    "@grpc/grpc-js": "^1.11.3",
+    "@grpc/proto-loader": "^0.7.13",
     "@ibm-generative-ai/node-sdk": "~3.2.3",
     "@langchain/community": "~0.2.28",
     "@langchain/core": "~0.2.27",
@@ -131,6 +134,8 @@
     "@eslint/js": "^9.9.0",
     "@eslint/markdown": "^6.0.0",
     "@googleapis/customsearch": "^3.2.0",
+    "@grpc/grpc-js": "^1.11.3",
+    "@grpc/proto-loader": "^0.7.13",
     "@ibm-generative-ai/node-sdk": "~3.2.3",
     "@langchain/community": "~0.2.28",
     "@langchain/core": "~0.2.27",
@@ -172,7 +177,7 @@
     "strip-ansi": "^7.1.0",
     "temp-dir": "^3.0.0",
     "tsc-files": "^1.1.4",
-    "tsup": "^8.2.4",
+    "tsup": "^8.3.0",
     "tsx": "^4.19.0",
     "typescript": "^5.5.4",
     "typescript-eslint": "^8.2.0",

scripts/copyright.sh

@@ -44,7 +44,7 @@ fi
 if command -v nwa &> /dev/null; then
   nwa add -l apache -c "$AUTHOR" src dist tests scripts
 elif command -v docker &> /dev/null; then
-  docker run -it -v "${PWD}:/src" ghcr.io/b1nary-gr0up/nwa:main add -l apache -c "$AUTHOR" src dist tests scripts
+  docker run --rm -v "${PWD}:/src" ghcr.io/b1nary-gr0up/nwa:main add -l apache -c "$AUTHOR" src dist tests scripts
 else
   echo "Error: 'nwa' is not available. Either install it manually or install go/docker."
   exit 1

scripts/ibm_vllm_generate_protos/ibm_vllm_generate_protos.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright 2024 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+GRPC_PROTO_PATH="./src/adapters/ibm-vllm/proto"
+GRPC_TYPES_PATH="./src/adapters/ibm-vllm/types.ts"
+
+SCRIPT_DIR="$(dirname "$0")"
+OUTPUT_RELATIVE_PATH="dist/generation.d.ts"
+GRPC_TYPES_TMP_PATH=types
+
+rm -f "$GRPC_TYPES_PATH"
+
+rm -rf "${SCRIPT_DIR}"/{dist,dts,types}
+
+
+yarn run proto-loader-gen-types \
+  --defaults \
+  --keepCase \
+  --oneofs \
+  --longs=Number \
+  --enums=String \
+  --grpcLib=@grpc/grpc-js \
+  --"outDir=${SCRIPT_DIR}/${GRPC_TYPES_TMP_PATH}" \
+  "${GRPC_PROTO_PATH}"/*.proto
+
+
+cd "$SCRIPT_DIR"
+  tsup --dts-only
+  sed -i.bak '$ d' "$OUTPUT_RELATIVE_PATH"
+  sed -i.bak -E "s/^interface/export interface/" "$OUTPUT_RELATIVE_PATH"
+  sed -i.bak -E "s/^type/export type/" "$OUTPUT_RELATIVE_PATH"
+cd -
+
+mv "$SCRIPT_DIR/$OUTPUT_RELATIVE_PATH" "$GRPC_TYPES_PATH"
+rm -rf "${SCRIPT_DIR}"/{dist,dts,types}
+
+yarn run lint:fix "${GRPC_TYPES_PATH}"
+yarn prettier --write "${GRPC_TYPES_PATH}"
+yarn copyright

scripts/ibm_vllm_generate_protos/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "ibm-vllm-proto-types",
+  "type": "module",
+  "version": "1.0.0",
+  "typings": "./types/generation.d.ts"
+}

scripts/ibm_vllm_generate_protos/tsconfig.proto.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node",
+    "rootDir": ".",
+    "baseUrl": ".",
+    "target": "ESNext",
+    "module": "ES6",
+    "outDir": "dist",
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "skipLibCheck": true,
+    "sourceMap": false
+  }
+}

scripts/ibm_vllm_generate_protos/tsup.config.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2024 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+  entry: ["types/generation.ts"],
+  tsconfig: "./tsconfig.proto.json",
+  sourcemap: false,
+  dts: true,
+  format: ["esm"],
+  treeshake: false,
+  legacyOutput: false,
+  skipNodeModulesBundle: true,
+  bundle: true,
+  splitting: false,
+  silent: false,
+  clean: true,
+});