From ef70c015472052f3f1ebfd6b6bfa2b00024dd7c9 Mon Sep 17 00:00:00 2001 From: Mateusz Kozak Date: Tue, 23 May 2023 21:01:54 +0200 Subject: [PATCH 1/4] added starcoder support --- examples/configured/code.yaml | 11 ++ poetry.lock | 144 +++++++++++++++++++- prr/services/providers/bigcode/starcoder.py | 56 ++++++++ prr/services/service_registry.py | 2 + pyproject.toml | 2 + 5 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 examples/configured/code.yaml create mode 100644 prr/services/providers/bigcode/starcoder.py diff --git a/examples/configured/code.yaml b/examples/configured/code.yaml new file mode 100644 index 0000000..7b498d3 --- /dev/null +++ b/examples/configured/code.yaml @@ -0,0 +1,11 @@ +version: 1 +prompt: + content: "def prompt_text_from_template(self, template):" +services: + models: + - 'bigcode/starcoder/starcoder' + options: + temperature: 0.7 + max_tokens: 250 + top_p: 0.9 + top_k: 40 \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 3cd7b03..f0205b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -411,7 +411,7 @@ files = [ name = "filelock" version = "3.12.0" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -507,6 +507,42 @@ files = [ {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] +[[package]] +name = "fsspec" +version = "2023.5.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.5.0-py3-none-any.whl", hash = "sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a"}, + {file = "fsspec-2023.5.0.tar.gz", hash = "sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + [[package]] name = "google-api-core" version = "2.11.0" @@ -969,6 +1005,38 @@ cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (>=1.0.0,<2.0.0)"] +[[package]] +name = "huggingface-hub" +version = "0.14.1" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, + {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, +] + +[package.dependencies] +filelock = "*" +fsspec = "*" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +torch = ["torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] + [[package]] name = "identify" version = "2.5.24" @@ -1390,6 +1458,59 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.6.0" +[[package]] +name = "pydantic" +version = "1.10.7" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d"}, + {file = "pydantic-1.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"}, + {file = "pydantic-1.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca"}, + {file = "pydantic-1.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d"}, + {file = "pydantic-1.10.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a"}, + {file = "pydantic-1.10.7-cp37-cp37m-win_amd64.whl", hash = "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209"}, + {file = "pydantic-1.10.7-cp38-cp38-win_amd64.whl", hash = "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5"}, + {file = "pydantic-1.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e"}, + {file = "pydantic-1.10.7-py3-none-any.whl", hash = "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6"}, + {file = "pydantic-1.10.7.tar.gz", hash = "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pygments" version = "2.15.1" @@ -1639,6 +1760,23 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "text-generation" +version = "0.5.2" +description = "Hugging Face Text Generation Python Client" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "text-generation-0.5.2.tar.gz", hash = "sha256:298da0ea72b1ad4e28bb6aadaa9addf4b05fda1ee5c0aa8d243b6831abc9f3d3"}, + {file = "text_generation-0.5.2-py3-none-any.whl", hash = "sha256:110dd5b05f8ec9cef00f3977001f1ce0464b133b14d4939b15260b0e863c942d"}, +] + +[package.dependencies] +aiohttp = ">=3.8,<4.0" +huggingface-hub = ">=0.12,<1.0" +pydantic = ">=1.10,<2.0" + [[package]] name = "tokenizers" version = "0.13.3" @@ -1731,7 +1869,7 @@ telegram = ["requests"] name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1868,4 +2006,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f5de07d0249fdb242cb75674b72e3194d9d84ecfecc132309143c80b63b8539f" +content-hash = "40e25e0f7357db311b62324afcd3877d7ab4f140ad218014cff3d37c9094ac53" diff --git a/prr/services/providers/bigcode/starcoder.py b/prr/services/providers/bigcode/starcoder.py new file mode 100644 index 0000000..59704eb --- /dev/null +++ b/prr/services/providers/bigcode/starcoder.py @@ -0,0 +1,56 @@ +from huggingface_hub import Repository +from text_generation import Client + +from prr.runner.request import ServiceRequest +from prr.runner.response import ServiceResponse +from prr.utils.config import load_config + +config = load_config() + +HF_TOKEN = config.get("HF_TOKEN", None) +API_URL = "https://api-inference.huggingface.co/models/bigcode/starcoder" + +FIM_PREFIX = "" +FIM_MIDDLE = "" +FIM_SUFFIX = "" + +FIM_INDICATOR = "" + + +class ServiceBigcodeStarcoder: + provider = "bigcode" + service = "starcoder" + + def run(self, prompt, service_config): + self.service_config = service_config + options = self.service_config.options + self.prompt = prompt + + client = Client( + API_URL, + headers={"Authorization": f"Bearer {HF_TOKEN}"}, + ) + + options = self.service_config.options + + prompt_text = prompt.template_text() + + service_request = ServiceRequest(service_config, prompt_text) + + response = client.generate( + prompt_text, + temperature=options.temperature, + max_new_tokens=options.max_tokens, + top_p=options.top_p, + # repetition_penalty=options.repetition_penalty, + ) + + service_response = ServiceResponse( + response.generated_text, + { + "tokens_used": response.details.generated_tokens, + "stop_reason": response.details.finish_reason, + }, + ) + + return service_request, service_response diff --git a/prr/services/service_registry.py b/prr/services/service_registry.py index 2149477..84516ce 100644 --- a/prr/services/service_registry.py +++ b/prr/services/service_registry.py @@ -2,6 +2,7 @@ from prr.services.providers.google.chat import ServiceGoogleChat from prr.services.providers.google.complete import ServiceGoogleComplete from prr.services.providers.openai.chat import ServiceOpenAIChat +from prr.services.providers.bigcode.starcoder import ServiceBigcodeStarcoder # main registry, where services are being... registered @@ -17,6 +18,7 @@ def register(self, service_class): def register_all_services(self): self.register(ServiceOpenAIChat) self.register(ServiceAnthropicComplete) + self.register(ServiceBigcodeStarcoder) self.register(ServiceGoogleComplete) self.register(ServiceGoogleChat) diff --git a/pyproject.toml b/pyproject.toml index 70470f9..8dbb182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ rich = "^13.3.5" Jinja2 = "^3.1.2" pyyaml = "^6.0" google-cloud-aiplatform = "^1.25.0" +huggingface-hub = "^0.14.1" +text-generation = "^0.5.2" [tool.poetry.dev-dependencies] pre-commit = "^2.12.0" From eca22af25332dce1087940cb2d4aba5cd858900e Mon Sep 17 00:00:00 2001 From: Mateusz Kozak Date: Tue, 23 May 2023 21:17:59 +0200 Subject: [PATCH 2/4] isort --- prr/services/service_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prr/services/service_registry.py b/prr/services/service_registry.py index 84516ce..1a2d2e3 100644 --- a/prr/services/service_registry.py +++ b/prr/services/service_registry.py @@ -1,8 +1,8 @@ from prr.services.providers.anthropic.complete import ServiceAnthropicComplete +from prr.services.providers.bigcode.starcoder import ServiceBigcodeStarcoder from prr.services.providers.google.chat import ServiceGoogleChat from prr.services.providers.google.complete import ServiceGoogleComplete from prr.services.providers.openai.chat import ServiceOpenAIChat -from prr.services.providers.bigcode.starcoder import ServiceBigcodeStarcoder # main registry, where services are being... registered From 28a9e13de7528bcb32976fd7fc56ebf4830b1a9b Mon Sep 17 00:00:00 2001 From: Mateusz Kozak Date: Mon, 29 May 2023 12:30:46 +0200 Subject: [PATCH 3/4] info on starcoder configuration and quick usage --- README.md | 21 ++++++++++++++++--- .../code.yaml => code/completion.yaml} | 0 2 files changed, 18 insertions(+), 3 deletions(-) rename examples/{configured/code.yaml => code/completion.yaml} (100%) diff --git a/README.md b/README.md index 9c241d1..2a9b219 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,9 @@ Everyone is welcome to contribute! As this is early stage work, there's lots improvements that can be done in the future and you're welcome to contribute! - [x] Get rix of Python 3.10 dependency -- [ ] Clean basic code smells +- [x] Clean basic code smells - [ ] Improve support for OpenAI and Anthropic -- [ ] Add support for other LLM providers +- [x] Add support for other LLM providers - [ ] Add support for locally hosted models - [ ] Pass model-related options to templating engine to allow for model-specific prompts - [ ] Add support for testing against expectations (elapsed_time, tokens_used) @@ -122,6 +122,14 @@ gcloud config set project gcloud auth application-default login ``` + +### Code completion +Using Starcoder model you can get code completion for a variety of languages. Here's a quick example of how to use it (check out the content of `examples/code/completion.yaml`): + +```sh +$ prr run ./examples/code/completion.yaml +``` + ### Run a prompt from a simple text file containing just a prompt Let's create a simple text file and call it `dingo` with the following content: @@ -181,7 +189,7 @@ If you refer to another template within your template, changes to that file will If your prompt is often saved and you're worried of running it too often, you can use `-c` option that's specific to `watch` command which enables defined number of seconds cooldown after every run, before it proceeds to execute on your changes again. ``` -$ ./prr watch -c 15 ./subconcepts-of-buddhism +$ prr watch -c 15 ./subconcepts-of-buddhism ``` @@ -418,6 +426,8 @@ stats: * OpenAI/chat - https://platform.openai.com/docs/guides/chat * Anthropic/complete - https://console.anthropic.com/docs/api +* Google Vertex AI PaLM - https://cloud.google.com/vertex-ai/docs/generative-ai/learn/overview +* Starcoder - https://huggingface.co/bigcode/starcoder ## Development @@ -450,6 +460,11 @@ OPENAI_API_KEY="sk-..." ANTHROPIC_API_KEY="sk-ant-..." DEFAULT_SERVICE="openai/chat/gpt-3.5-turbo" +# https://console.cloud.google.com +GOOGLE_PROJECT="gcp-project-id" +GOOGLE_LOCATION="us-central1" +# https://huggingface.co/settings/tokens +HF_TOKEN="hf_..." ``` You can also use DEFAULT_SERVICE to specify the model you want to use by default, but otherwise you're good to go! diff --git a/examples/configured/code.yaml b/examples/code/completion.yaml similarity index 100% rename from examples/configured/code.yaml rename to examples/code/completion.yaml From 31c3a4c8511eda8605acf8c399a2e395ff75ff31 Mon Sep 17 00:00:00 2001 From: Mateusz Kozak Date: Tue, 30 May 2023 12:48:42 +0200 Subject: [PATCH 4/4] remove doubled line --- prr/services/providers/bigcode/starcoder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/prr/services/providers/bigcode/starcoder.py b/prr/services/providers/bigcode/starcoder.py index 59704eb..256a900 100644 --- a/prr/services/providers/bigcode/starcoder.py +++ b/prr/services/providers/bigcode/starcoder.py @@ -23,7 +23,6 @@ class ServiceBigcodeStarcoder: def run(self, prompt, service_config): self.service_config = service_config - options = self.service_config.options self.prompt = prompt client = Client(