Merge pull request fermyon#1790 from karthik2804/refactor-llm

Refactor llm and add http compute option
itowlson · Sep 22, 2023 · 17976d3 · 17976d3
2 parents 75c8110 + 5b8e01c
commit 17976d3
Show file tree

Hide file tree

Showing 14 changed files with 768 additions and 460 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/llm-local/Cargo.toml b/crates/llm-local/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "spin-llm-local"
+version = { workspace = true }
+authors = { workspace = true }
+edition = { workspace = true }
+
+[dependencies]
+anyhow = "1.0"
+candle = { git = "https://github.com/huggingface/candle", rev = "b80348d22f8f0dadb6cc4101bde031d5de69a9a5", package = "candle-core" }
+candle-nn = { git = "https://github.com/huggingface/candle", rev = "b80348d22f8f0dadb6cc4101bde031d5de69a9a5" }
+chrono = "0.4.26"
+llm = { git = "https://github.com/rustformers/llm", rev = "2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663", features = [
+    "tokenizers-remote",
+    "llama",
+], default-features = false }
+lru = "0.9.0"
+num_cpus = "1"
+rand = "0.8.5"
+safetensors = "0.3.3"
+serde = { version = "1.0.150", features = ["derive"] }
+spin-core = { path = "../core" }
+spin-llm = { path = "../llm"}
+spin-world = { path = "../world" }
+terminal = { path = "../terminal" }
+tokenizers = "0.13.4"
+tokio = { version = "1.32.0", features = ["macros", "sync"] }
+tracing = { workspace = true }
+
+[features]
+default = []
+metal = ["llm/metal"]
+cublas = ["llm/cublas"]
diff --git a/crates/llm/src/bert.rs → crates/llm-local/src/bert.rs b/crates/llm/src/bert.rs → crates/llm-local/src/bert.rs